Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Make the first channel of channels.txt being treated again, solve temporary empty response from YouTube Data API v3 issue and temporarily remove sanity check failing very rarely #39

Browse Source
This commit is contained in:
Benjamin Loison
2023-02-14 23:15:07 +01:00
parent e166fdb4e5
commit cb7b68342a
1 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
main.cpp
View File

@@ -117,13 +117,16 @@ int main(int argc, char *argv[])
for(const auto& entry : filesystem::directory_iterator(CHANNELS_DIRECTORY)) for(const auto& entry : filesystem::directory_iterator(CHANNELS_DIRECTORY))
{ {
string fileName = entry.path().filename(), string fileName = entry.path().filename();
channelId = fileName.substr(0, fileName.length() - 4); // Skip files such as `UNLISTED_VIDEOS_FILE_PATH`.
if (fileName.substr(0, 2) == "UC") {
string channelId = fileName.substr(0, fileName.length() - 4);
channelsToTreat.erase(channelsToTreatRev[channelId]); channelsToTreat.erase(channelsToTreatRev[channelId]);
channelsToTreatRev.erase(channelId); channelsToTreatRev.erase(channelId);
channelsAlreadyTreated.insert(channelId); channelsAlreadyTreated.insert(channelId);
}
} }
MAIN_PRINT(channelsToTreat.size() << " channel(s) to treat") MAIN_PRINT(channelsToTreat.size() << " channel(s) to treat")
@@ -532,7 +535,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
// We verify that we don't skip any message by verifying that the first message was already treated if we already treated some messages. // We verify that we don't skip any message by verifying that the first message was already treated if we already treated some messages.
if(!messageIds.empty() && messageIds.find(firstMessageId) == messageIds.end()) if(!messageIds.empty() && messageIds.find(firstMessageId) == messageIds.end())
{ {
EXIT_WITH_ERROR("The verification that we don't skip any message failed!") PRINT("The verification that we don't skip any message failed! Continuing anyway...")
} }
for(const auto& message : snippet) for(const auto& message : snippet)
{ {
@@ -751,7 +754,9 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
} }
catch (json::parse_error& ex) catch (json::parse_error& ex)
{ {
EXIT_WITH_ERROR("Parse error for " << finalUrl << ", as got: " << content << " !") // From the experience this sometimes happens due to empty `content` but retrying just after solves the problem.
PRINT("Parse error for " << finalUrl << ", as got: " << content << " ! Retrying...")
return getJson(threadId, url, usingYoutubeDataApiv3, channelId);
} }
if(data.contains("error")) if(data.contains("error"))