Day 21/100 100 Days of Code
I implemented mutex in my program and moved the code around to fix some problems. Now, the critical sections get locked and the next thread is waiting for the thread that has locked the section to unlock it. I also fixed an issue where canceling` the operation didn't stop all the threads.
void MainFrame::StartScraping(int amount, int counter, std::vector<std::string> keywords,
std::vector<std::string> getUrls)
{
if (Scraper::isCanceled)
{
if (!threads.empty())
{
threads.clear();
}
return;
}
if (!Scraper::CheckForConnection())
{
wxMessageBox("You have been disconnected from the internet",
"", wxOK);
Scraper::isCanceled = true;
if (scrapingInfoText != nullptr)
{
scrapingInfoText->Destroy();
scrapingInfoText = nullptr;
}
if (!threads.empty())
{
threads.clear();
}
return;
}
m.lock();
std::vector<std::string> scraperKeywords;
scraperKeywords.reserve(amount);
for (int j = 0; j < amount; j++)
{
scraperKeywords.push_back(keywords[j]);
}
if (scrapingInfoText != nullptr)
{
scrapingInfoText->Destroy();
scrapingInfoText = nullptr;
}
content->SetFont(wxFontInfo(32).FaceName("Helvetica Neue").Bold());
Scraper::SetupScraper(scraperKeywords, getUrls[counter]);
scrapingInfoText = new wxStaticText(MainFrame::content, wxID_ANY,
std::string("Currently checking: ") +
std::string(getUrls[counter]),
wxDefaultPosition, wxDefaultSize);
scrapingInfoSizer = new wxBoxSizer(wxVERTICAL);
scrapingInfoSizer->Add(scrapingInfoText, 0, wxCENTER);
runContentHolder->Add(scrapingInfoSizer, 1, wxEXPAND);
content->SetSizer(runContentHolder);
content->Layout();
// Get info from website
cpr::Response r = Scraper::request_info(Scraper::baseURL);
// Parse it
std::vector<std::string> urls = Scraper::ParseContent(r.text, (char *) "href",
(char *) "/");
// Iterate through the urls
for (const std::string &item: urls) {
if (!Scraper::CheckForConnection())
{
wxMessageBox("You have been disconnected from the internet", "",
wxOK);
if (scrapingInfoText != nullptr)
{
scrapingInfoText->Destroy();
scrapingInfoText = nullptr;
}
scrapingState = SST_Waiting;
m.unlock();
return;
}
AnalyzePages::analyzeEntry(item, scraperKeywords, scraper);
}
if (scrapingInfoText != nullptr)
{
scrapingInfoText->Destroy();
scrapingInfoText = nullptr;
}
m.unlock();
if (Scraper::isCanceled && operationCounter == operationSize)
{
wxMessageBox("Operation has been canceled.", "",wxOK);
operationCounter = 0;
operationSize = 0;
if (!threads.empty())
{
threads.clear();
}
}
else if (operationCounter == operationSize)
{
if (!threads.empty())
{
threads.clear();
}
operationCounter = 0;
operationSize = 0;
if (scrapingInfoText != nullptr)
{
scrapingInfoText->Destroy();
scrapingInfoText = nullptr;
}
scrapingState = SST_Waiting;
wxMessageBox("Operation has been completed.", "", wxOK);
} else
{
operationCounter++;
}
}
Subscribe to my newsletter
Read articles from Chris Douris directly inside your inbox. Subscribe to the newsletter, and don't miss out.
Written by
Chris Douris
Chris Douris
AKA Chris, is a software developer from Athens, Greece. He started programming with basic when he was very young. He lost interest in programming during school years but after an unsuccessful career in audio, he decided focus on what he really loves which is technology. He loves working with older languages like C and wants to start programming electronics and microcontrollers because he wants to get into embedded systems programming.