Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Make PRINT not requiring to precise threadId

Browse Source
This commit is contained in:
Benjamin Loison 2023-01-22 02:04:03 +01:00
parent 548a797ee8
commit 59dc5676cc
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
main.cpp
View File

@ -30,9 +30,10 @@ size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath),
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
#define THREAD_PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
#define PRINT(x) THREAD_PRINT(threadId, x)
#define DEFAULT_THREAD_ID 0
#define MAIN_PRINT(x) PRINT(DEFAULT_THREAD_ID, x)
#define MAIN_PRINT(x) THREAD_PRINT(DEFAULT_THREAD_ID, x)
mutex printMutex,
channelsAlreadyTreatedAndToTreatMutex,
@ -160,7 +161,7 @@ void treatChannels(unsigned short threadId)
string channelToTreat = channelsToTreat.begin()->second;
PRINT(threadId, "Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
PRINT("Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
commentsCountThreads[threadId] = 0;
requestsPerChannelThreads[threadId] = 0;
@ -178,15 +179,15 @@ void treatChannels(unsigned short threadId)
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
PRINT(threadId, "Starting compression...")
PRINT("Starting compression...")
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@");
PRINT(threadId, "Compression finished, started deleting initial directory...")
PRINT("Compression finished, started deleting initial directory...")
deleteDirectory(channelToTreatDirectory);
PRINT(threadId, "Deleting directory finished.")
PRINT("Deleting directory finished.")
PRINT(threadId, commentsCountThreads[threadId] << " comments were found for this channel.")
PRINT(commentsCountThreads[threadId] << " comments were found for this channel.")
}
channelsAlreadyTreatedAndToTreatMutex.unlock();
@ -255,11 +256,11 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
}
else
{
PRINT(threadId, "Comments disabled channel, treating differently...")
PRINT("Comments disabled channel, treating differently...")
json data = getJson(threadId, "channels?part=statistics&id=" + channelToTreat, true, channelToTreat);
// YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer...
unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str());
PRINT(threadId, "The channel has about " << videoCount << " videos.")
PRINT("The channel has about " << videoCount << " videos.")
// `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here.
if(0 < videoCount && videoCount < 20000)
{
@ -271,7 +272,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound);
if(data.contains("error"))
{
PRINT(threadId, "Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
PRINT("Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
exit(EXIT_FAILURE);
}
json items = data["items"];
@ -295,12 +296,12 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
}
else if(videoCount == 0)
{
PRINT(threadId, "Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.")
PRINT("Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.")
break;
}
else //if(videoCount >= 20000)
{
PRINT(threadId, "The videos count of the channel exceeds the supported 20,000 limit!")
PRINT("The videos count of the channel exceeds the supported 20,000 limit!")
exit(EXIT_FAILURE);
}
}
@ -437,7 +438,7 @@ breakChannelsTreatment:
{
string videoId = snippet["resourceId"]["videoId"],
channelId = snippet["videoOwnerChannelId"];
PRINT(threadId, "Found non public video (" << videoId << ") in: " << playlistId)
PRINT("Found non public video (" << videoId << ") in: " << playlistId)
string channelUnlistedVideosFilePath = CHANNELS_DIRECTORY + UNLISTED_VIDEOS_FILE_PATH;
bool doesChannelUnlistedVideosFileExist = doesFileExist(channelUnlistedVideosFilePath);
writeFile(threadId, channelUnlistedVideosFilePath, !doesChannelUnlistedVideosFileExist ? "w" : "a", (!doesChannelUnlistedVideosFileExist ? "" : "\n") + channelId);
@ -536,7 +537,7 @@ bool writeFile(unsigned short threadId, string filePath, string option, string t
}
else
{
PRINT(threadId, "writeFile error: " << strerror(errno))
PRINT("writeFile error: " << strerror(errno))
}
return false;
}
@ -600,7 +601,7 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
}
catch (json::parse_error& ex)
{
PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content << " !")
PRINT("Parse error for " << finalUrl << ", as got: " << content << " !")
exit(EXIT_FAILURE);
}
@ -613,12 +614,12 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
quotaMutex.lock();
keys.erase(keys.begin());
keys.push_back(apiKey);
PRINT(threadId, "No more quota on " << apiKey << " switching to " << keys[0] << ".")
PRINT("No more quota on " << apiKey << " switching to " << keys[0] << ".")
apiKey = keys[0];
quotaMutex.unlock();
return getJson(threadId, url, true, directoryPath);
}
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
PRINT("Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
{
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath);