From 8df226e2bcfde559db7e4729893e3d7a854b54f1 Mon Sep 17 00:00:00 2001 From: Benjamin Loison Date: Fri, 10 Feb 2023 20:17:49 +0100 Subject: [PATCH] Move YouTube API requests logging to `requests/` channel sub-folder --- main.cpp | 84 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/main.cpp b/main.cpp index a2db4c9..0a90700 100644 --- a/main.cpp +++ b/main.cpp @@ -16,7 +16,7 @@ enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNot set setFromVector(vector vec); vector getFileContent(string filePath); -json getJson(unsigned short threadId, string url, bool usingYouTubeDataApiV3, string directoryPath, getJsonBehavior behavior = normal); +json getJson(unsigned short threadId, string url, bool usingYouTubeDataApiV3, string channelId, getJsonBehavior behavior = normal); void createDirectory(string path), print(ostringstream* toPrint), treatComment(unsigned short threadId, json comment, string channelId), @@ -59,7 +59,8 @@ string CHANNELS_DIRECTORY = "channels/", apiKey = "", // Will firstly be filled with `KEYS_FILE_PATH` first line. YOUTUBE_OPERATIONAL_API_INSTANCE_URL = "http://localhost/YouTube-operational-API", // Can be "https://yt.lemnoslife.com" for instance. CAPTIONS_DIRECTORY = "captions/", - DEBUG_DIRECTORY = "debug/"; + DEBUG_DIRECTORY = "debug/", + YOUTUBE_API_REQUESTS_DIRECTORY = "requests/"; bool USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE = false; int main(int argc, char *argv[]) @@ -181,6 +182,7 @@ void treatChannels(unsigned short threadId) createDirectory(channelToTreatDirectory); createDirectory(DEBUG_DIRECTORY); createDirectory(channelToTreatDirectory + CAPTIONS_DIRECTORY); + createDirectory(channelToTreatDirectory + YOUTUBE_API_REQUESTS_DIRECTORY); treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat); @@ -562,45 +564,45 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str break; } } - } - // Captions retrieval by relying on `yt-dlp` after having listed all videos ids of the given channel. - string playlistToTreat = "UU" + channelToTreat.substr(2); - pageToken = ""; - while(true) - { - json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound); - if(data.contains("error")) + // Captions retrieval by relying on `yt-dlp` after having listed all videos ids of the given channel. + string playlistToTreat = "UU" + channelToTreat.substr(2); + pageToken = ""; + while(true) { - EXIT_WITH_ERROR("Not listing captions on videos, as `playlistItems` hasn't found the `uploads` playlist!") - } - json items = data["items"]; - for(const auto& item : items) - { - string videoId = item["contentDetails"]["videoId"]; - // Could proceed as follows by verifying `!isChannel` but as we don't know how to manage unlisted videos, we don't proceed this way. - //treatChannelOrVideo(threadId, false, videoId, channelToTreat); + json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound); + if(data.contains("error")) + { + EXIT_WITH_ERROR("Not listing captions on videos, as `playlistItems` hasn't found the `uploads` playlist!") + } + json items = data["items"]; + for(const auto& item : items) + { + string videoId = item["contentDetails"]["videoId"]; + // Could proceed as follows by verifying `!isChannel` but as we don't know how to manage unlisted videos, we don't proceed this way. + //treatChannelOrVideo(threadId, false, videoId, channelToTreat); - string channelCaptionsToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/" + CAPTIONS_DIRECTORY + videoId + "/"; - createDirectory(channelCaptionsToTreatDirectory); + string channelCaptionsToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/" + CAPTIONS_DIRECTORY + videoId + "/"; + createDirectory(channelCaptionsToTreatDirectory); - // Firstly download all not automatically generated captions. - // The underscore in `-o` argument is used to not end up with hidden files. - string cmdCommonPrefix = "yt-dlp --skip-download ", - cmdCommonPostfix = " '" + videoId + "' -o '" + channelCaptionsToTreatDirectory + "_'"; - string cmd = cmdCommonPrefix + "--all-subs" + cmdCommonPostfix; - exec(threadId, cmd); + // Firstly download all not automatically generated captions. + // The underscore in `-o` argument is used to not end up with hidden files. + string cmdCommonPrefix = "yt-dlp --skip-download ", + cmdCommonPostfix = " '" + videoId + "' -o '" + channelCaptionsToTreatDirectory + "_'"; + string cmd = cmdCommonPrefix + "--all-subs" + cmdCommonPostfix; + exec(threadId, cmd); - // Secondly download the automatically generated captions. - cmd = cmdCommonPrefix + "--write-auto-subs --sub-langs '.*orig' --sub-format ttml --convert-subs vtt" + cmdCommonPostfix; - exec(threadId, cmd); - } - if(data.contains("nextPageToken")) - { - pageToken = data["nextPageToken"]; - } - else - { - break; + // Secondly download the automatically generated captions. + cmd = cmdCommonPrefix + "--write-auto-subs --sub-langs '.*orig' --sub-format ttml --convert-subs vtt" + cmdCommonPostfix; + exec(threadId, cmd); + } + if(data.contains("nextPageToken")) + { + pageToken = data["nextPageToken"]; + } + else + { + break; + } } } } @@ -734,7 +736,7 @@ vector getFileContent(string filePath) return lines; } -json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, string directoryPath, getJsonBehavior behavior) +json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, string channelId, getJsonBehavior behavior) { string finalUrl = usingYoutubeDataApiv3 ? (USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE ? @@ -768,17 +770,17 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st PRINT("No more quota on " << apiKey << " switching to " << keys[0] << ".") apiKey = keys[0]; quotaMutex.unlock(); - return getJson(threadId, url, true, directoryPath); + return getJson(threadId, url, true, channelId); } PRINT("Found error in JSON at URL: " << finalUrl << " for content: " << content << " !") if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled) { - return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath); + return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, channelId); } } ostringstream toString; - toString << CHANNELS_DIRECTORY << directoryPath << "/"; + toString << CHANNELS_DIRECTORY << channelId << "/" << YOUTUBE_API_REQUESTS_DIRECTORY; writeFile(threadId, toString.str() + "urls.txt", "a", url + " " + (usingYoutubeDataApiv3 ? "true" : "false") + "\n"); toString << requestsPerChannelThreads[threadId]++ << ".json"; writeFile(threadId, toString.str(), "w", content);