Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

#7: Make commentsCount and requestsPerChannel compatible with multithreading

Browse Source
This commit is contained in:
Benjamin Loison 2023-01-15 14:31:55 +01:00
parent 7456685f2b
commit f114aac0cf
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 12 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
main.cpp
View File

@ -42,9 +42,9 @@ set<string> channelsAlreadyTreated;
map<unsigned int, string> channelsToTreat; map<unsigned int, string> channelsToTreat;
map<string, unsigned int> channelsToTreatRev; map<string, unsigned int> channelsToTreatRev;
vector<string> keys; vector<string> keys;
unsigned int commentsCount = 0, unsigned int commentsPerSecondCount = 0;
commentsPerSecondCount = 0, map<unsigned short, unsigned int> commentsCountThreads,
requestsPerChannel = 0; requestsPerChannelThreads;
unsigned short THREADS_NUMBER = 1; unsigned short THREADS_NUMBER = 1;
string CHANNELS_DIRECTORY = "channels/", string CHANNELS_DIRECTORY = "channels/",
CHANNELS_FILE_PATH = "channels.txt", CHANNELS_FILE_PATH = "channels.txt",
@ -160,6 +160,9 @@ void treatChannels(unsigned short threadId)
PRINT(threadId, "Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")") PRINT(threadId, "Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
commentsCountThreads[threadId] = 0;
requestsPerChannelThreads[threadId] = 0;
channelsToTreat.erase(channelsToTreatRev[channelToTreat]); channelsToTreat.erase(channelsToTreatRev[channelToTreat]);
channelsToTreatRev.erase(channelToTreat); channelsToTreatRev.erase(channelToTreat);
@ -181,14 +184,13 @@ void treatChannels(unsigned short threadId)
deleteDirectory(channelToTreatDirectory); deleteDirectory(channelToTreatDirectory);
PRINT(threadId, "Deleting directory finished.") PRINT(threadId, "Deleting directory finished.")
PRINT(threadId, commentsCount << " comments were found for this channel.") PRINT(threadId, commentsCountThreads[threadId] << " comments were found for this channel.")
commentsCount = 0;
requestsPerChannel = 0;
} }
channelsAlreadyTreatedAndToTreatMutex.unlock(); channelsAlreadyTreatedAndToTreatMutex.unlock();
} }
// Have to pay attention not to recursively call this function with another channel otherwise we break the ability of the program to halt at any top level channel.
void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat) void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat)
{ {
string pageToken = ""; string pageToken = "";
@ -303,6 +305,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
} }
if(isChannel) if(isChannel)
{ {
// `CHANNELS`
string pageToken = ""; string pageToken = "";
while(true) while(true)
{ {
@ -357,7 +360,7 @@ void treatComment(unsigned short threadId, json comment, string channelId)
string channelId = snippet["authorChannelId"]["value"]; string channelId = snippet["authorChannelId"]["value"];
addChannelToTreat(threadId, channelId); addChannelToTreat(threadId, channelId);
} }
commentsCount++; commentsCountThreads[threadId]++;
commentsPerSecondCount++; commentsPerSecondCount++;
} }
@ -478,8 +481,8 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
} }
ostringstream toString; ostringstream toString;
toString << CHANNELS_DIRECTORY << directoryPath << "/" << requestsPerChannel << ".json"; toString << CHANNELS_DIRECTORY << directoryPath << "/" << requestsPerChannelThreads[threadId] << ".json";
requestsPerChannel++; requestsPerChannelThreads[threadId]++;
writeFile(threadId, toString.str(), "w", url + "\n" + content); writeFile(threadId, toString.str(), "w", url + "\n" + content);
return data; return data;