Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Fix #8: Support comments disabled channels

Browse Source 
Tested with `UCWIdqSQekeGmUWlSFeCiEnA` which treated correctly the 36 comments of the only comments enabled video `3F8dFt8LsXY`.

Note that this commit doesn't support comments disabled channels with more than 20,000 videos.
This commit is contained in:
Benjamin Loison 2023-01-03 02:56:07 +01:00
parent 923c14a77b
commit a2990c7699
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 107 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
main.cpp
View File

@ -12,7 +12,8 @@ vector<string> getFileContent(string filePath);
json getJson(string url, string directoryPath); json getJson(string url, string directoryPath);
void createDirectory(string path), void createDirectory(string path),
print(ostringstream* toPrint), print(ostringstream* toPrint),
treatComment(json comment, string channelId); treatComment(json comment, string channelId),
treatChannelOrVideo(bool isChannel, string id, string channelToTreat);
string getHttps(string url); string getHttps(string url);
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp); size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath), bool doesFileExist(string filePath),
@ -63,11 +64,28 @@ int main()
string channelToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/"; string channelToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/";
createDirectory(channelToTreatDirectory); createDirectory(channelToTreatDirectory);
treatChannelOrVideo(true, channelToTreat, channelToTreat);
PRINT(commentsCount)
commentsCount = 0;
requestsPerChannel = 0;
channelsToTreat.erase(channelToTreat);
}
return 0;
}
void treatChannelOrVideo(bool isChannel, string id, string channelToTreat)
{
string pageToken = ""; string pageToken = "";
while(true) while(true)
{ {
json data = getJson("commentThreads?part=snippet,replies&allThreadsRelatedToChannelId=" + channelToTreat + "&maxResults=100&pageToken=" + pageToken, channelToTreat); ostringstream toString;
bool doesRelyingOnCommentThreadsIsEnough = data["error"]["errors"][0]["reason"] != "commentsDisabled"; toString << "commentThreads?part=snippet,replies&" << (isChannel ? "allThreadsRelatedToChannelId" : "videoId") << "=" << id << "&maxResults=100&pageToken=" << pageToken;
string url = toString.str();
json data = getJson(url, channelToTreat);
bool doesRelyingOnCommentThreadsIsEnough = (!isChannel) || data["error"]["errors"][0]["reason"] != "commentsDisabled";
if(doesRelyingOnCommentThreadsIsEnough) if(doesRelyingOnCommentThreadsIsEnough)
{ {
json items = data["items"]; json items = data["items"];
@ -84,8 +102,8 @@ int main()
string pageToken = ""; string pageToken = "";
while(true) while(true)
{ {
json data = getJson("comments?part=snippet&parentId=" + commentId + "&maxResults=100&pageToken=" + pageToken, channelToTreat); json data = getJson("comments?part=snippet&parentId=" + commentId + "&maxResults=100&pageToken=" + pageToken, channelToTreat),
json items = data["items"]; items = data["items"];
for(const auto& item : items) for(const auto& item : items)
{ {
treatComment(item, channelToTreat); treatComment(item, channelToTreat);
@ -120,19 +138,46 @@ int main()
} }
else else
{ {
PRINT("Comments disabled channel!") PRINT("Comments disabled channel, treating differently...")
json data = getJson("channels?part=statistics&id=" + channelToTreat, channelToTreat);
// YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer...
unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str());
PRINT("The channel has about " << videoCount << " videos.")
// `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here.
if(videoCount < 20000)
{
string playlistToTreat = "UU" + channelToTreat.substr(2),
pageToken = "";
while(true)
{
// `snippet` and `status` are unneeded `part`s here but may be interesting later, as we log them.
json data = getJson("playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, channelToTreat),
items = data["items"];
for(const auto& item : items)
{
string videoId = item["contentDetails"]["videoId"];
// To keep the same amount of logs for each channel, I comment the following `PRINT`.
//PRINT("Treating video " << videoId)
treatChannelOrVideo(false, videoId, channelToTreat);
}
if(data.contains("nextPageToken"))
{
pageToken = data["nextPageToken"];
}
else
{
break;
}
}
break;
}
else
{
PRINT("The videos count of the channel exceeds the supported 20,000 limit!")
exit(1); exit(1);
} }
} }
PRINT(commentsCount)
commentsCount = 0;
requestsPerChannel = 0;
channelsToTreat.erase(channelToTreat);
} }
return 0;
} }
void treatComment(json comment, string channelId) void treatComment(json comment, string channelId)