Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Fix #20: YouTube Data API v3 returns rarely suddenly commentsDisabled error which involves an unwanted method switch

Browse Source 
Also modified compression command, as I got `sh: 1: zip: Argument list too long` when compressing the 248,868 json files of the French most subscribers channel.
This commit is contained in:
Benjamin Loison 2023-01-08 15:43:27 +01:00
parent 3ae0f4e924
commit b3779fe49a
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
2 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
main.cpp
View File

@ -13,7 +13,7 @@ using namespace chrono;
using json = nlohmann::json;
vector<string> getFileContent(string filePath);
json getJson(unsigned short threadId, string url, string directoryPath);
json getJson(unsigned short threadId, string url, string directoryPath, bool retryOnCommentsDisabled = false);
void createDirectory(string path),
print(ostringstream* toPrint),
treatComment(unsigned short threadId, json comment, string channelId),
@ -49,13 +49,15 @@ int main()
// To resume this algorithm after a shutdown, just restart it after having deleted the last channel folders in `CHANNELS_DIRECTORY` being treated.
// On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set.
vector<string> channelsVec = getFileContent(CHANNELS_FILE_PATH);
// Note that using `set`s makes the search faster but we lose the `channels.txt` lines order.
channelsToTreat = set(channelsVec.begin(), channelsVec.end());
createDirectory(CHANNELS_DIRECTORY);
for(const auto& entry : filesystem::directory_iterator(CHANNELS_DIRECTORY))
{
string channelId = entry.path().filename();
string fileName = entry.path().filename(),
channelId = fileName.substr(0, fileName.length() - 4);
channelsToTreat.erase(channelId);
channelsAlreadyTreated.insert(channelId);
}
@ -112,9 +114,14 @@ void treatChannels(unsigned short threadId)
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
PRINT(threadId, "Starting compression...")
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
exec("cd " + channelToTreatDirectory + " && zip -r ../" + channelToTreat + ".zip *");
exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@");
PRINT(threadId, "Compression finished, started deleting initial directory...")
deleteDirectory(channelToTreatDirectory);
PRINT(threadId, "Deleting directory finished.")
PRINT(threadId, commentsCount << " comments were found for this channel.")
commentsCount = 0;
@ -132,7 +139,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
ostringstream toString;
toString << "commentThreads?part=snippet,replies&" << (isChannel ? "allThreadsRelatedToChannelId" : "videoId") << "=" << id << "&maxResults=100&pageToken=" << pageToken;
string url = toString.str();
json data = getJson(threadId, url, channelToTreat);
json data = getJson(threadId, url, channelToTreat, pageToken != "");
bool doesRelyingOnCommentThreadsIsEnough = (!isChannel) || data["error"]["errors"][0]["reason"] != "commentsDisabled";
if(doesRelyingOnCommentThreadsIsEnough)
{
@ -323,7 +330,7 @@ vector<string> getFileContent(string filePath)
return lines;
}
json getJson(unsigned short threadId, string url, string directoryPath)
json getJson(unsigned short threadId, string url, string directoryPath, bool retryOnCommentsDisabled)
{
#ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
string finalUrl = "https://yt.lemnoslife.com/noKey/" + url;
@ -345,8 +352,11 @@ json getJson(unsigned short threadId, string url, string directoryPath)
if(data.contains("error"))
{
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
if(data["error"]["errors"][0]["reason"] != "commentsDisabled" || retryOnCommentsDisabled)
{
return getJson(threadId, url, directoryPath);
}
}
ostringstream toString;
toString << CHANNELS_DIRECTORY << directoryPath << "/" << requestsPerChannel << ".json";

10
removeChannelsBeingTreated.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/python3
import shutil
import shutil, os
infix = ': Treating channel '
path = 'channels/'
@ -18,8 +18,16 @@ with open('nohup.out') as f:
for threadId in threads:
channelId = threads[threadId]
print(threadId, channelId)
# There are three cases:
# - `channelId`/ exists
# - `channelId`/ and `channelId`.zip exist
# - `channelId`.zip exists
# To manage every case, we need to use two `try`/`except`.
try:
shutil.rmtree(path + channelId)
except:
pass
try:
os.remove(path + channelId + ".zip")
except:
pass