Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

#7: Remove remaining undefined behavior due to missing mutex use

Browse Source
This commit is contained in:
Benjamin Loison 2023-01-06 18:00:51 +01:00
parent 3ef5fa0707
commit fdfec17817
1 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
main.cpp
View File

@ -34,7 +34,7 @@ bool doesFileExist(string filePath),
#define DEFAULT_THREAD_ID 0 #define DEFAULT_THREAD_ID 0
mutex printMutex, mutex printMutex,
allocateChannelMutex; channelsAlreadyTreatedAndToTreatMutex;
set<string> channelsAlreadyTreated, set<string> channelsAlreadyTreated,
channelsToTreat; channelsToTreat;
unsigned int commentsCount = 0, unsigned int commentsCount = 0,
@ -90,10 +90,10 @@ void treatChannels(unsigned short threadId)
// For the moment we assume that we never have treated completely YouTube, otherwise we have to pay attention how to proceed if the starting set involves startvation for some threads. // For the moment we assume that we never have treated completely YouTube, otherwise we have to pay attention how to proceed if the starting set involves startvation for some threads.
while(true) while(true)
{ {
allocateChannelMutex.lock(); channelsAlreadyTreatedAndToTreatMutex.lock();
if(channelsToTreat.empty()) if(channelsToTreat.empty())
{ {
allocateChannelMutex.unlock(); channelsAlreadyTreatedAndToTreatMutex.unlock();
sleep(1); sleep(1);
continue; continue;
} }
@ -105,7 +105,7 @@ void treatChannels(unsigned short threadId)
channelsToTreat.erase(channelToTreat); channelsToTreat.erase(channelToTreat);
channelsAlreadyTreated.insert(channelToTreat); channelsAlreadyTreated.insert(channelToTreat);
allocateChannelMutex.unlock(); channelsAlreadyTreatedAndToTreatMutex.unlock();
string channelToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/"; string channelToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/";
createDirectory(channelToTreatDirectory); createDirectory(channelToTreatDirectory);
@ -121,7 +121,7 @@ void treatChannels(unsigned short threadId)
requestsPerChannel = 0; requestsPerChannel = 0;
} }
allocateChannelMutex.unlock(); channelsAlreadyTreatedAndToTreatMutex.unlock();
} }
void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat) void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat)
@ -144,8 +144,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
treatComment(threadId, comment, channelToTreat); treatComment(threadId, comment, channelToTreat);
if(item.contains("replies")) if(item.contains("replies"))
{ {
json replies = item["replies"]["comments"]; if(item["snippet"]["totalReplyCount"] > 5)
if(replies.size() >= 5)
{ {
string pageToken = ""; string pageToken = "";
while(true) while(true)
@ -168,6 +167,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
} }
else else
{ {
json replies = item["replies"]["comments"];
for(const auto& reply : replies) for(const auto& reply : replies)
{ {
treatComment(threadId, reply, channelToTreat); treatComment(threadId, reply, channelToTreat);
@ -235,12 +235,18 @@ void treatComment(unsigned short threadId, json comment, string channelId)
if(snippet.contains("authorChannelId")) if(snippet.contains("authorChannelId"))
{ {
string channelId = snippet["authorChannelId"]["value"]; string channelId = snippet["authorChannelId"]["value"];
if(find(channelsAlreadyTreated.begin(), channelsAlreadyTreated.end(), channelId) == channelsAlreadyTreated.end() && find(channelsToTreat.begin(), channelsToTreat.end(), channelId) == channelsToTreat.end()) channelsAlreadyTreatedAndToTreatMutex.lock();
if(channelsAlreadyTreated.find(channelId) == channelsAlreadyTreated.end() && channelsToTreat.find(channelId) == channelsToTreat.end())
{ {
channelsToTreat.insert(channelId); channelsToTreat.insert(channelId);
channelsAlreadyTreatedAndToTreatMutex.unlock();
writeFile(threadId, CHANNELS_FILE_PATH, "a", "\n" + channelId); writeFile(threadId, CHANNELS_FILE_PATH, "a", "\n" + channelId);
} }
else
{
channelsAlreadyTreatedAndToTreatMutex.unlock();
}
} }
commentsCount++; commentsCount++;
commentsPerSecondCount++; commentsPerSecondCount++;
@ -332,7 +338,7 @@ json getJson(unsigned short threadId, string url, string directoryPath)
} }
catch (json::parse_error& ex) catch (json::parse_error& ex)
{ {
PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content) PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content << " !")
exit(1); exit(1);
} }