diff --git a/main.cpp b/main.cpp index de45d58..5a79c70 100644 --- a/main.cpp +++ b/main.cpp @@ -34,7 +34,7 @@ bool doesFileExist(string filePath), #define DEFAULT_THREAD_ID 0 mutex printMutex, - allocateChannelMutex; + channelsAlreadyTreatedAndToTreatMutex; set channelsAlreadyTreated, channelsToTreat; unsigned int commentsCount = 0, @@ -90,10 +90,10 @@ void treatChannels(unsigned short threadId) // For the moment we assume that we never have treated completely YouTube, otherwise we have to pay attention how to proceed if the starting set involves startvation for some threads. while(true) { - allocateChannelMutex.lock(); + channelsAlreadyTreatedAndToTreatMutex.lock(); if(channelsToTreat.empty()) { - allocateChannelMutex.unlock(); + channelsAlreadyTreatedAndToTreatMutex.unlock(); sleep(1); continue; } @@ -105,7 +105,7 @@ void treatChannels(unsigned short threadId) channelsToTreat.erase(channelToTreat); channelsAlreadyTreated.insert(channelToTreat); - allocateChannelMutex.unlock(); + channelsAlreadyTreatedAndToTreatMutex.unlock(); string channelToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/"; createDirectory(channelToTreatDirectory); @@ -121,7 +121,7 @@ void treatChannels(unsigned short threadId) requestsPerChannel = 0; } - allocateChannelMutex.unlock(); + channelsAlreadyTreatedAndToTreatMutex.unlock(); } void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat) @@ -144,8 +144,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str treatComment(threadId, comment, channelToTreat); if(item.contains("replies")) { - json replies = item["replies"]["comments"]; - if(replies.size() >= 5) + if(item["snippet"]["totalReplyCount"] > 5) { string pageToken = ""; while(true) @@ -168,6 +167,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str } else { + json replies = item["replies"]["comments"]; for(const auto& reply : replies) { treatComment(threadId, reply, channelToTreat); @@ -235,12 +235,18 @@ void treatComment(unsigned short threadId, json comment, string channelId) if(snippet.contains("authorChannelId")) { string channelId = snippet["authorChannelId"]["value"]; - if(find(channelsAlreadyTreated.begin(), channelsAlreadyTreated.end(), channelId) == channelsAlreadyTreated.end() && find(channelsToTreat.begin(), channelsToTreat.end(), channelId) == channelsToTreat.end()) + channelsAlreadyTreatedAndToTreatMutex.lock(); + if(channelsAlreadyTreated.find(channelId) == channelsAlreadyTreated.end() && channelsToTreat.find(channelId) == channelsToTreat.end()) { channelsToTreat.insert(channelId); + channelsAlreadyTreatedAndToTreatMutex.unlock(); writeFile(threadId, CHANNELS_FILE_PATH, "a", "\n" + channelId); } + else + { + channelsAlreadyTreatedAndToTreatMutex.unlock(); + } } commentsCount++; commentsPerSecondCount++; @@ -332,7 +338,7 @@ json getJson(unsigned short threadId, string url, string directoryPath) } catch (json::parse_error& ex) { - PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content) + PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content << " !") exit(1); }