Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Make PRINT not requiring to precise threadId

Browse Source
This commit is contained in:
Benjamin Loison 2023-01-22 02:04:03 +01:00
parent 548a797ee8
commit 59dc5676cc
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
main.cpp
View File

@ -30,9 +30,10 @@ size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath), bool doesFileExist(string filePath),
writeFile(unsigned short threadId, string filePath, string option, string toWrite); writeFile(unsigned short threadId, string filePath, string option, string toWrite);
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); } #define THREAD_PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
#define PRINT(x) THREAD_PRINT(threadId, x)
#define DEFAULT_THREAD_ID 0 #define DEFAULT_THREAD_ID 0
#define MAIN_PRINT(x) PRINT(DEFAULT_THREAD_ID, x) #define MAIN_PRINT(x) THREAD_PRINT(DEFAULT_THREAD_ID, x)
mutex printMutex, mutex printMutex,
channelsAlreadyTreatedAndToTreatMutex, channelsAlreadyTreatedAndToTreatMutex,
@ -160,7 +161,7 @@ void treatChannels(unsigned short threadId)
string channelToTreat = channelsToTreat.begin()->second; string channelToTreat = channelsToTreat.begin()->second;
PRINT(threadId, "Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")") PRINT("Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
commentsCountThreads[threadId] = 0; commentsCountThreads[threadId] = 0;
requestsPerChannelThreads[threadId] = 0; requestsPerChannelThreads[threadId] = 0;
@ -178,15 +179,15 @@ void treatChannels(unsigned short threadId)
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat); treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds. // Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
PRINT(threadId, "Starting compression...") PRINT("Starting compression...")
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli. // As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@"); exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@");
PRINT(threadId, "Compression finished, started deleting initial directory...") PRINT("Compression finished, started deleting initial directory...")
deleteDirectory(channelToTreatDirectory); deleteDirectory(channelToTreatDirectory);
PRINT(threadId, "Deleting directory finished.") PRINT("Deleting directory finished.")
PRINT(threadId, commentsCountThreads[threadId] << " comments were found for this channel.") PRINT(commentsCountThreads[threadId] << " comments were found for this channel.")
} }
channelsAlreadyTreatedAndToTreatMutex.unlock(); channelsAlreadyTreatedAndToTreatMutex.unlock();
@ -255,11 +256,11 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
} }
else else
{ {
PRINT(threadId, "Comments disabled channel, treating differently...") PRINT("Comments disabled channel, treating differently...")
json data = getJson(threadId, "channels?part=statistics&id=" + channelToTreat, true, channelToTreat); json data = getJson(threadId, "channels?part=statistics&id=" + channelToTreat, true, channelToTreat);
// YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer... // YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer...
unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str()); unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str());
PRINT(threadId, "The channel has about " << videoCount << " videos.") PRINT("The channel has about " << videoCount << " videos.")
// `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here. // `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here.
if(0 < videoCount && videoCount < 20000) if(0 < videoCount && videoCount < 20000)
{ {
@ -271,7 +272,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound); json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound);
if(data.contains("error")) if(data.contains("error"))
{ {
PRINT(threadId, "Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!") PRINT("Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
json items = data["items"]; json items = data["items"];
@ -295,12 +296,12 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
} }
else if(videoCount == 0) else if(videoCount == 0)
{ {
PRINT(threadId, "Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.") PRINT("Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.")
break; break;
} }
else //if(videoCount >= 20000) else //if(videoCount >= 20000)
{ {
PRINT(threadId, "The videos count of the channel exceeds the supported 20,000 limit!") PRINT("The videos count of the channel exceeds the supported 20,000 limit!")
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
@ -437,7 +438,7 @@ breakChannelsTreatment:
{ {
string videoId = snippet["resourceId"]["videoId"], string videoId = snippet["resourceId"]["videoId"],
channelId = snippet["videoOwnerChannelId"]; channelId = snippet["videoOwnerChannelId"];
PRINT(threadId, "Found non public video (" << videoId << ") in: " << playlistId) PRINT("Found non public video (" << videoId << ") in: " << playlistId)
string channelUnlistedVideosFilePath = CHANNELS_DIRECTORY + UNLISTED_VIDEOS_FILE_PATH; string channelUnlistedVideosFilePath = CHANNELS_DIRECTORY + UNLISTED_VIDEOS_FILE_PATH;
bool doesChannelUnlistedVideosFileExist = doesFileExist(channelUnlistedVideosFilePath); bool doesChannelUnlistedVideosFileExist = doesFileExist(channelUnlistedVideosFilePath);
writeFile(threadId, channelUnlistedVideosFilePath, !doesChannelUnlistedVideosFileExist ? "w" : "a", (!doesChannelUnlistedVideosFileExist ? "" : "\n") + channelId); writeFile(threadId, channelUnlistedVideosFilePath, !doesChannelUnlistedVideosFileExist ? "w" : "a", (!doesChannelUnlistedVideosFileExist ? "" : "\n") + channelId);
@ -536,7 +537,7 @@ bool writeFile(unsigned short threadId, string filePath, string option, string t
} }
else else
{ {
PRINT(threadId, "writeFile error: " << strerror(errno)) PRINT("writeFile error: " << strerror(errno))
} }
return false; return false;
} }
@ -600,7 +601,7 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
} }
catch (json::parse_error& ex) catch (json::parse_error& ex)
{ {
PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content << " !") PRINT("Parse error for " << finalUrl << ", as got: " << content << " !")
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@ -613,12 +614,12 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
quotaMutex.lock(); quotaMutex.lock();
keys.erase(keys.begin()); keys.erase(keys.begin());
keys.push_back(apiKey); keys.push_back(apiKey);
PRINT(threadId, "No more quota on " << apiKey << " switching to " << keys[0] << ".") PRINT("No more quota on " << apiKey << " switching to " << keys[0] << ".")
apiKey = keys[0]; apiKey = keys[0];
quotaMutex.unlock(); quotaMutex.unlock();
return getJson(threadId, url, true, directoryPath); return getJson(threadId, url, true, directoryPath);
} }
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !") PRINT("Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled) if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
{ {
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath); return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath);