Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Compare commits

base: Benjamin_Loison:10e88118173045f2f672d4d7117c55080dbdad92
Branches Tags
Benjamin_Loison:master
Benjamin_Loison:0.0.1
Benjamin_Loison:0.0.2
Benjamin_Loison:0.0.0
..
compare: Benjamin_Loison:bdb4e6443acc52aee1736c60096050d2b8f7c2a0
Branches Tags
Benjamin_Loison:master
Benjamin_Loison:0.0.1
Benjamin_Loison:0.0.2
Benjamin_Loison:0.0.0

No commits in common. "10e88118173045f2f672d4d7117c55080dbdad92" and "bdb4e6443acc52aee1736c60096050d2b8f7c2a0" have entirely different histories.
10e8811817 ... bdb4e6443a

1 changed files with 14 additions and 47 deletions
Show Stats Expand all files Collapse all files

61
main.cpp
View File

@ -36,9 +36,6 @@ bool doesFileExist(string filePath),
#define DEFAULT_THREAD_ID 0 #define DEFAULT_THREAD_ID 0
#define MAIN_PRINT(x) THREAD_PRINT(DEFAULT_THREAD_ID, x) #define MAIN_PRINT(x) THREAD_PRINT(DEFAULT_THREAD_ID, x)
#define EXIT_WITH_ERROR(x) { PRINT(x); exit(EXIT_FAILURE); }
#define MAIN_EXIT_WITH_ERROR(x) { MAIN_PRINT(x); exit(EXIT_FAILURE); }
mutex printMutex, mutex printMutex,
channelsAlreadyTreatedAndToTreatMutex, channelsAlreadyTreatedAndToTreatMutex,
quotaMutex; quotaMutex;
@ -87,12 +84,14 @@ int main(int argc, char *argv[])
} }
else else
{ {
MAIN_EXIT_WITH_ERROR("YouTube operational API instance URL missing!") MAIN_PRINT("YouTube operational API instance URL missing!")
exit(EXIT_FAILURE);
} }
} }
else else
{ {
MAIN_EXIT_WITH_ERROR("Unrecognized parameter " << argvStr) MAIN_PRINT("Unrecognized parameter " << argvStr)
exit(EXIT_FAILURE);
} }
} }
@ -274,7 +273,8 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound); json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound);
if(data.contains("error")) if(data.contains("error"))
{ {
EXIT_WITH_ERROR("Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!") PRINT("Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
exit(EXIT_FAILURE);
} }
json items = data["items"]; json items = data["items"];
for(const auto& item : items) for(const auto& item : items)
@ -302,7 +302,8 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
} }
else //if(videoCount >= 20000) else //if(videoCount >= 20000)
{ {
EXIT_WITH_ERROR("The videos count of the channel exceeds the supported 20,000 limit!") PRINT("The videos count of the channel exceeds the supported 20,000 limit!")
exit(EXIT_FAILURE);
} }
} }
} }
@ -487,11 +488,11 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
{ {
if(item.contains("liveStreamingDetails")) if(item.contains("liveStreamingDetails"))
{ {
string videoId = item["id"]; PRINT(item["id"])
//PRINT(videoId)
json liveStreamingDetails = item["liveStreamingDetails"]; json liveStreamingDetails = item["liveStreamingDetails"];
if(liveStreamingDetails.contains("activeLiveChatId")) if(liveStreamingDetails.contains("activeLiveChatId"))
{ {
PRINT("streaming")
string activeLiveChatId = liveStreamingDetails["activeLiveChatId"]; string activeLiveChatId = liveStreamingDetails["activeLiveChatId"];
json data = getJson(threadId, "liveChat/messages?part=snippet,authorDetails&liveChatId=" + activeLiveChatId, true, id), json data = getJson(threadId, "liveChat/messages?part=snippet,authorDetails&liveChatId=" + activeLiveChatId, true, id),
items = data["items"]; items = data["items"];
@ -499,47 +500,12 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
{ {
string channelId = item["snippet"]["authorChannelId"]; string channelId = item["snippet"]["authorChannelId"];
addChannelToTreat(threadId, channelId); addChannelToTreat(threadId, channelId);
PRINT("Found: " << channelId)
} }
} }
else else
{ {
// As there isn't the usual pagination mechanism for these ended livestreams, we proceed in an uncertain way as follows. PRINT("no more streaming")
set<string> messageIds;
unsigned long long lastMessageTimestampRelativeMsec = 0;
while(true)
{
string time = to_string(lastMessageTimestampRelativeMsec);
json data = getJson(threadId, "liveChats?part=snippet&id=" + videoId + "&time=" + time, false, id),
snippet = data["items"][0]["snippet"];
if(snippet.empty())
{
break;
}
json firstMessage = snippet[0];
string firstMessageId = firstMessage["id"];
// We verify that we don't skip any message by verifying that the first message was already treated if we already treated some messages.
if(!messageIds.empty() && messageIds.find(firstMessageId) == messageIds.end())
{
EXIT_WITH_ERROR("The verification that we don't skip any message failed!")
}
for(const auto& message : snippet)
{
string messageId = message["id"];
if(messageIds.find(messageId) == messageIds.end())
{
messageIds.insert(messageId);
string channelId = message["authorChannelId"];
addChannelToTreat(threadId, channelId);
}
}
json lastMessage = snippet.back();
// If there isn't any new message, then we stop the retrieving.
if(lastMessageTimestampRelativeMsec == lastMessage["videoOffsetTimeMsec"])
{
break;
}
lastMessageTimestampRelativeMsec = lastMessage["videoOffsetTimeMsec"];
}
} }
} }
} }
@ -696,7 +662,8 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
} }
catch (json::parse_error& ex) catch (json::parse_error& ex)
{ {
EXIT_WITH_ERROR("Parse error for " << finalUrl << ", as got: " << content << " !") PRINT("Parse error for " << finalUrl << ", as got: " << content << " !")
exit(EXIT_FAILURE);
} }
if(data.contains("error")) if(data.contains("error"))