Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues16 Pull Requests Packages Projects Releases2 Wiki Activity

Make the first channel of channels.txt being treated again, solve temporary empty response from YouTube Data API v3 issue and temporarily remove sanity check failing very rarely

Browse Source
This commit is contained in:
Benjamin Loison 2023-02-14 23:15:07 +01:00
parent e166fdb4e5
commit cb7b68342a
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 12 additions and 7 deletions

19
main.cpp

@ -117,13 +117,16 @@ int main(int argc, char *argv[])
for(const auto& entry : filesystem::directory_iterator(CHANNELS_DIRECTORY)) for(const auto& entry : filesystem::directory_iterator(CHANNELS_DIRECTORY))
{ {
string fileName = entry.path().filename(), string fileName = entry.path().filename();
channelId = fileName.substr(0, fileName.length() - 4); // Skip files such as `UNLISTED_VIDEOS_FILE_PATH`.
if (fileName.substr(0, 2) == "UC") {
string channelId = fileName.substr(0, fileName.length() - 4);
channelsToTreat.erase(channelsToTreatRev[channelId]); channelsToTreat.erase(channelsToTreatRev[channelId]);
channelsToTreatRev.erase(channelId); channelsToTreatRev.erase(channelId);
channelsAlreadyTreated.insert(channelId); channelsAlreadyTreated.insert(channelId);
}
} }
MAIN_PRINT(channelsToTreat.size() << " channel(s) to treat") MAIN_PRINT(channelsToTreat.size() << " channel(s) to treat")
@ -532,7 +535,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
// We verify that we don't skip any message by verifying that the first message was already treated if we already treated some messages. // We verify that we don't skip any message by verifying that the first message was already treated if we already treated some messages.
if(!messageIds.empty() && messageIds.find(firstMessageId) == messageIds.end()) if(!messageIds.empty() && messageIds.find(firstMessageId) == messageIds.end())
{ {
EXIT_WITH_ERROR("The verification that we don't skip any message failed!") PRINT("The verification that we don't skip any message failed! Continuing anyway...")
} }
for(const auto& message : snippet) for(const auto& message : snippet)
{ {
@ -751,7 +754,9 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
} }
catch (json::parse_error& ex) catch (json::parse_error& ex)
{ {
EXIT_WITH_ERROR("Parse error for " << finalUrl << ", as got: " << content << " !") // From the experience this sometimes happens due to empty `content` but retrying just after solves the problem.
PRINT("Parse error for " << finalUrl << ", as got: " << content << " ! Retrying...")
return getJson(threadId, url, usingYoutubeDataApiv3, channelId);
} }
if(data.contains("error")) if(data.contains("error"))