761 lines
31 KiB
C++
761 lines
31 KiB
C++
#include <iostream>
|
|
#include <fstream>
|
|
#include <sstream>
|
|
#include <set>
|
|
#include <mutex>
|
|
#include <thread>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
#include <curl/curl.h>
|
|
#include <nlohmann/json.hpp>
|
|
using namespace std;
|
|
using namespace chrono;
|
|
using json = nlohmann::json;
|
|
|
|
enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNotFound };
|
|
|
|
set<string> setFromVector(vector<string> vec);
|
|
vector<string> getFileContent(string filePath);
|
|
json getJson(unsigned short threadId, string url, bool usingYouTubeDataApiV3, string directoryPath, getJsonBehavior behavior = normal);
|
|
void createDirectory(string path),
|
|
print(ostringstream* toPrint),
|
|
treatComment(unsigned short threadId, json comment, string channelId),
|
|
treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat),
|
|
treatChannels(unsigned short threadId),
|
|
deleteDirectory(string path),
|
|
addChannelToTreat(unsigned short threadId, string channelId);
|
|
string getHttps(string url),
|
|
exec(string cmd),
|
|
join(vector<string> parts, string delimiter);
|
|
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
|
|
bool doesFileExist(string filePath),
|
|
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
|
|
|
#define THREAD_PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
|
|
#define PRINT(x) THREAD_PRINT(threadId, x)
|
|
#define DEFAULT_THREAD_ID 0
|
|
#define MAIN_PRINT(x) THREAD_PRINT(DEFAULT_THREAD_ID, x)
|
|
|
|
#define EXIT_WITH_ERROR(x) { PRINT(x); exit(EXIT_FAILURE); }
|
|
#define MAIN_EXIT_WITH_ERROR(x) { MAIN_PRINT(x); exit(EXIT_FAILURE); }
|
|
|
|
mutex printMutex,
|
|
channelsAlreadyTreatedAndToTreatMutex,
|
|
quotaMutex;
|
|
set<string> channelsAlreadyTreated;
|
|
// Two `map`s to simulate a bidirectional map.
|
|
map<unsigned int, string> channelsToTreat;
|
|
map<string, unsigned int> channelsToTreatRev;
|
|
vector<string> keys;
|
|
unsigned int commentsPerSecondCount = 0;
|
|
map<unsigned short, unsigned int> commentsCountThreads,
|
|
requestsPerChannelThreads;
|
|
unsigned short THREADS_NUMBER = 1;
|
|
// Use `string` variables instead of macros to have `string` properties, even if could use a meta-macro inlining as `string`s.
|
|
string CHANNELS_DIRECTORY = "channels/",
|
|
CHANNELS_FILE_PATH = "channels.txt",
|
|
KEYS_FILE_PATH = "keys.txt",
|
|
UNLISTED_VIDEOS_FILE_PATH = "unlistedVideos.txt",
|
|
apiKey = "", // Will firstly be filled with `KEYS_FILE_PATH` first line.
|
|
YOUTUBE_OPERATIONAL_API_INSTANCE_URL = "http://localhost/YouTube-operational-API"; // Can be "https://yt.lemnoslife.com" for instance.
|
|
bool USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE = false;
|
|
|
|
int main(int argc, char *argv[])
|
|
{
|
|
for(unsigned short argvIndex = 1; argvIndex < argc; argvIndex++)
|
|
{
|
|
string argvStr = string(argv[argvIndex]);
|
|
if(argvStr == "--no-keys")
|
|
{
|
|
USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE = true;
|
|
}
|
|
else if(argvStr.rfind("--threads=", 0) == 0)
|
|
{
|
|
THREADS_NUMBER = atoi(argvStr.substr(10).c_str());
|
|
}
|
|
else if(argvStr == "-h" || argvStr == "--help")
|
|
{
|
|
MAIN_PRINT("Usage: " << argv[0] << " [--help/-h] [--no-keys] [--threads=N] [--youtube-operational-api-instance-url URL]")
|
|
exit(EXIT_SUCCESS);
|
|
}
|
|
else if(argvStr == "--youtube-operational-api-instance-url")
|
|
{
|
|
if(argvIndex < argc - 1)
|
|
{
|
|
YOUTUBE_OPERATIONAL_API_INSTANCE_URL = string(argv[argvIndex + 1]);
|
|
argvIndex++;
|
|
}
|
|
else
|
|
{
|
|
MAIN_EXIT_WITH_ERROR("YouTube operational API instance URL missing!")
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MAIN_EXIT_WITH_ERROR("Unrecognized parameter " << argvStr)
|
|
}
|
|
}
|
|
|
|
// The starting set should be written to `CHANNELS_FILE_PATH`.
|
|
// To resume this algorithm after a shutdown, just restart it after having deleted the last channel folders in `CHANNELS_DIRECTORY` being treated.
|
|
// On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat*` or `channelsToTreat*` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set.
|
|
vector<string> channelsVec = getFileContent(CHANNELS_FILE_PATH);
|
|
for(unsigned int channelsVecIndex = 0; channelsVecIndex < channelsVec.size(); channelsVecIndex++)
|
|
{
|
|
string channel = channelsVec[channelsVecIndex];
|
|
channelsToTreat[channelsVecIndex] = channel;
|
|
channelsToTreatRev[channel] = channelsVecIndex;
|
|
}
|
|
|
|
keys = getFileContent(KEYS_FILE_PATH);
|
|
apiKey = keys[0];
|
|
|
|
createDirectory(CHANNELS_DIRECTORY);
|
|
|
|
for(const auto& entry : filesystem::directory_iterator(CHANNELS_DIRECTORY))
|
|
{
|
|
string fileName = entry.path().filename(),
|
|
channelId = fileName.substr(0, fileName.length() - 4);
|
|
|
|
channelsToTreat.erase(channelsToTreatRev[channelId]);
|
|
channelsToTreatRev.erase(channelId);
|
|
|
|
channelsAlreadyTreated.insert(channelId);
|
|
}
|
|
|
|
MAIN_PRINT(channelsToTreat.size() << " channel(s) to treat")
|
|
MAIN_PRINT(channelsAlreadyTreated.size() << " channel(s) already treated")
|
|
|
|
vector<thread> threads;
|
|
for(unsigned short threadsIndex = 0; threadsIndex < THREADS_NUMBER; threadsIndex++)
|
|
{
|
|
threads.push_back(thread(treatChannels, threadsIndex + 1));
|
|
}
|
|
|
|
while(true)
|
|
{
|
|
MAIN_PRINT("Comments per second: " << commentsPerSecondCount)
|
|
commentsPerSecondCount = 0;
|
|
sleep(1);
|
|
}
|
|
|
|
// The following is dead code, as we assume below not to have ever treated completely YouTube.
|
|
for(unsigned short threadsIndex = 0; threadsIndex < THREADS_NUMBER; threadsIndex++)
|
|
{
|
|
threads[threadsIndex].join();
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void treatChannels(unsigned short threadId)
|
|
{
|
|
// For the moment we assume that we never have treated completely YouTube, otherwise we have to pay attention how to proceed if the starting set involves startvation for some threads.
|
|
while(true)
|
|
{
|
|
channelsAlreadyTreatedAndToTreatMutex.lock();
|
|
if(channelsToTreat.empty())
|
|
{
|
|
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
|
sleep(1);
|
|
continue;
|
|
}
|
|
|
|
string channelToTreat = channelsToTreat.begin()->second;
|
|
|
|
PRINT("Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
|
|
|
|
commentsCountThreads[threadId] = 0;
|
|
requestsPerChannelThreads[threadId] = 0;
|
|
|
|
channelsToTreat.erase(channelsToTreatRev[channelToTreat]);
|
|
channelsToTreatRev.erase(channelToTreat);
|
|
|
|
channelsAlreadyTreated.insert(channelToTreat);
|
|
|
|
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
|
|
|
string channelToTreatDirectory = CHANNELS_DIRECTORY + channelToTreat + "/";
|
|
createDirectory(channelToTreatDirectory);
|
|
|
|
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
|
|
|
|
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
|
|
PRINT("Starting compression...")
|
|
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
|
|
exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@");
|
|
|
|
PRINT("Compression finished, started deleting initial directory...")
|
|
deleteDirectory(channelToTreatDirectory);
|
|
PRINT("Deleting directory finished.")
|
|
|
|
PRINT(commentsCountThreads[threadId] << " comments were found for this channel.")
|
|
}
|
|
|
|
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
|
}
|
|
|
|
// Have to pay attention not to recursively call this function with another channel otherwise we break the ability of the program to halt at any top level channel.
|
|
void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat)
|
|
{
|
|
string pageToken = "";
|
|
while(true)
|
|
{
|
|
ostringstream toString;
|
|
toString << "commentThreads?part=snippet,replies&" << (isChannel ? "allThreadsRelatedToChannelId" : "videoId") << "=" << id << "&maxResults=100&pageToken=" << pageToken;
|
|
string url = toString.str();
|
|
json data = getJson(threadId, url, true, channelToTreat, pageToken == "" ? normal : retryOnCommentsDisabled);
|
|
bool doesRelyingOnCommentThreadsIsEnough = (!isChannel) || data["error"]["errors"][0]["reason"] != "commentsDisabled";
|
|
if(doesRelyingOnCommentThreadsIsEnough)
|
|
{
|
|
json items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
json comment = item["snippet"]["topLevelComment"];
|
|
string commentId = comment["id"];
|
|
treatComment(threadId, comment, channelToTreat);
|
|
if(item.contains("replies"))
|
|
{
|
|
if(item["snippet"]["totalReplyCount"] > 5)
|
|
{
|
|
string pageToken = "";
|
|
while(true)
|
|
{
|
|
json data = getJson(threadId, "comments?part=snippet&parentId=" + commentId + "&maxResults=100&pageToken=" + pageToken, true, channelToTreat),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
treatComment(threadId, item, channelToTreat);
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
json replies = item["replies"]["comments"];
|
|
for(const auto& reply : replies)
|
|
{
|
|
treatComment(threadId, reply, channelToTreat);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
PRINT("Comments disabled channel, treating differently...")
|
|
json data = getJson(threadId, "channels?part=statistics&id=" + channelToTreat, true, channelToTreat);
|
|
// YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer...
|
|
unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str());
|
|
PRINT("The channel has about " << videoCount << " videos.")
|
|
// `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here.
|
|
if(0 < videoCount && videoCount < 20000)
|
|
{
|
|
string playlistToTreat = "UU" + channelToTreat.substr(2),
|
|
pageToken = "";
|
|
while(true)
|
|
{
|
|
// `snippet` and `status` are unneeded `part`s here but may be interesting later, as we log them.
|
|
json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound);
|
|
if(data.contains("error"))
|
|
{
|
|
EXIT_WITH_ERROR("Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
|
|
}
|
|
json items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
string videoId = item["contentDetails"]["videoId"];
|
|
// To keep the same amount of logs for each channel, I comment the following `PRINT`.
|
|
//PRINT("Treating video " << videoId)
|
|
treatChannelOrVideo(threadId, false, videoId, channelToTreat);
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
else if(videoCount == 0)
|
|
{
|
|
PRINT("Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.")
|
|
break;
|
|
}
|
|
else //if(videoCount >= 20000)
|
|
{
|
|
EXIT_WITH_ERROR("The videos count of the channel exceeds the supported 20,000 limit!")
|
|
}
|
|
}
|
|
}
|
|
if(isChannel)
|
|
{
|
|
// `CHANNELS`
|
|
string pageToken = "";
|
|
while(true)
|
|
{
|
|
json data = getJson(threadId, "channels?part=channels&id=" + id + (pageToken == "" ? "" : "&pageToken=" + pageToken), false, id),
|
|
channelSections = data["items"][0]["channelSections"];
|
|
for(const auto& channelSection : channelSections)
|
|
{
|
|
for(const auto& sectionChannel : channelSection["sectionChannels"])
|
|
{
|
|
string channelId = sectionChannel["channelId"];
|
|
addChannelToTreat(threadId, channelId);
|
|
}
|
|
}
|
|
if(channelSections.size() == 1)
|
|
{
|
|
json channelSection = channelSections[0];
|
|
if(!channelSection["nextPageToken"].is_null())
|
|
{
|
|
pageToken = channelSection["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
// `COMMUNITY`
|
|
pageToken = "";
|
|
while(true)
|
|
{
|
|
json data = getJson(threadId, "channels?part=community&id=" + id + (pageToken == "" ? "" : "&pageToken=" + pageToken), false, id);
|
|
data = data["items"][0];
|
|
json posts = data["community"];
|
|
for(const auto& post : posts)
|
|
{
|
|
string postId = post["id"];
|
|
json data = getJson(threadId, "community?part=snippet&id=" + postId + "&order=time", false, id);
|
|
string pageToken = data["items"][0]["snippet"]["comments"]["nextPageToken"];
|
|
while(pageToken != "")
|
|
{
|
|
json data = getJson(threadId, "commentThreads?part=snippet,replies&pageToken=" + pageToken, false, id),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
json snippet = item["snippet"]["topLevelComment"]["snippet"];
|
|
string channelId = snippet["authorChannelId"]["value"];
|
|
addChannelToTreat(threadId, channelId);
|
|
string pageToken = snippet["nextPageToken"];
|
|
while(pageToken != "")
|
|
{
|
|
json data = getJson(threadId, "commentThreads?part=snippet,replies&pageToken=" + pageToken, false, id),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
string channelId = item["snippet"]["authorChannelId"]["value"];
|
|
addChannelToTreat(threadId, channelId);
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
if(pageToken == "")
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
// `PLAYLISTS`
|
|
pageToken = "";
|
|
while(true)
|
|
{
|
|
json data = getJson(threadId, "channels?part=playlists&id=" + id + (pageToken == "" ? "" : "&pageToken=" + pageToken), false, id),
|
|
playlistSections = data["items"][0]["playlistSections"];
|
|
|
|
for(const auto& playlistSection : playlistSections)
|
|
{
|
|
for(const auto& playlist : playlistSection["playlists"])
|
|
{
|
|
string playlistId = playlist["id"];
|
|
//PRINT(threadId, playlistId)
|
|
string pageToken = "";
|
|
while(true)
|
|
{
|
|
json data = getJson(threadId, "playlistItems?part=contentDetails,snippet,status&playlistId=" + playlistId + "&maxResults=50&pageToken=" + pageToken, true, id),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
json snippet = item["snippet"];
|
|
string privacyStatus = item["status"]["privacyStatus"];
|
|
// `5-CXVU8si3A` in `PLTYUE9O6WCrjQsnOm56rMMNmFy_A-SjUx` has its privacy status on `privacyStatusUnspecified` and is inaccessible.
|
|
// `GMiVi8xkEXA` in `PLTYUE9O6WCrgNpeSiryP8LYVX-7tOJ1f1` has its privacy status on `private`.
|
|
// Of course `commentThreads?videoId=` doesn't work for these videos (same result on YouTube UI).
|
|
// By hypothesis that the discovery algorithm never ends we can't postpone the treatment of these unlisted videos, because we can find such unlisted videos at any point in time (before or after the given channel treatment).
|
|
// Maybe modifying this hypothesis would make sense, otherwise we have to treat them right-away (note that except code architecture, there is no recursion problem as documented on this function).
|
|
if(privacyStatus != "public" && privacyStatus != "private" && snippet["title"] != "Deleted video")
|
|
{
|
|
string videoId = snippet["resourceId"]["videoId"],
|
|
channelId = snippet["videoOwnerChannelId"];
|
|
PRINT("Found non public video (" << videoId << ") in: " << playlistId)
|
|
string channelUnlistedVideosFilePath = CHANNELS_DIRECTORY + UNLISTED_VIDEOS_FILE_PATH;
|
|
bool doesChannelUnlistedVideosFileExist = doesFileExist(channelUnlistedVideosFilePath);
|
|
writeFile(threadId, channelUnlistedVideosFilePath, !doesChannelUnlistedVideosFileExist ? "w" : "a", (!doesChannelUnlistedVideosFileExist ? "" : "\n") + channelId);
|
|
}
|
|
if(snippet.contains("videoOwnerChannelId"))
|
|
{
|
|
// There isn't any `videoOwnerChannelId` to retrieve for `5-CXVU8si3A` for instance.
|
|
string channelId = snippet["videoOwnerChannelId"];
|
|
if(channelId != id)
|
|
{
|
|
addChannelToTreat(threadId, channelId);
|
|
}
|
|
}
|
|
}
|
|
if(data.contains("nextPageToken"))
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if(!data["nextPageToken"].is_null())
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
// `LIVE`
|
|
pageToken = "";
|
|
string playlistId = "UU" + id.substr(2);
|
|
vector<string> videoIds;
|
|
while(true)
|
|
{
|
|
json data = getJson(threadId, "playlistItems?part=contentDetails,snippet,status&playlistId=" + playlistId + "&maxResults=50&pageToken=" + pageToken, true, id, returnErrorIfPlaylistNotFound),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
string videoId = item["snippet"]["resourceId"]["videoId"];
|
|
videoIds.push_back(videoId);
|
|
}
|
|
bool hasNextPageToken = data.contains("nextPageToken");
|
|
if(videoIds.size() == 50 || !hasNextPageToken)
|
|
{
|
|
json data = getJson(threadId, "videos?part=contentDetails,id,liveStreamingDetails,localizations,player,snippet,statistics,status,topicDetails&id=" + join(videoIds, ","), true, id),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
if(item.contains("liveStreamingDetails"))
|
|
{
|
|
string videoId = item["id"];
|
|
//PRINT(videoId)
|
|
json liveStreamingDetails = item["liveStreamingDetails"];
|
|
if(liveStreamingDetails.contains("activeLiveChatId"))
|
|
{
|
|
string activeLiveChatId = liveStreamingDetails["activeLiveChatId"];
|
|
json data = getJson(threadId, "liveChat/messages?part=snippet,authorDetails&liveChatId=" + activeLiveChatId, true, id),
|
|
items = data["items"];
|
|
for(const auto& item : items)
|
|
{
|
|
string channelId = item["snippet"]["authorChannelId"];
|
|
addChannelToTreat(threadId, channelId);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// As there isn't the usual pagination mechanism for these ended livestreams, we proceed in an uncertain way as follows.
|
|
set<string> messageIds;
|
|
unsigned long long lastMessageTimestampRelativeMsec = 0;
|
|
while(true)
|
|
{
|
|
string time = to_string(lastMessageTimestampRelativeMsec);
|
|
json data = getJson(threadId, "liveChats?part=snippet&id=" + videoId + "&time=" + time, false, id),
|
|
snippet = data["items"][0]["snippet"];
|
|
if(snippet.empty())
|
|
{
|
|
break;
|
|
}
|
|
json firstMessage = snippet[0];
|
|
string firstMessageId = firstMessage["id"];
|
|
// We verify that we don't skip any message by verifying that the first message was already treated if we already treated some messages.
|
|
if(!messageIds.empty() && messageIds.find(firstMessageId) == messageIds.end())
|
|
{
|
|
EXIT_WITH_ERROR("The verification that we don't skip any message failed!")
|
|
}
|
|
for(const auto& message : snippet)
|
|
{
|
|
string messageId = message["id"];
|
|
if(messageIds.find(messageId) == messageIds.end())
|
|
{
|
|
messageIds.insert(messageId);
|
|
string channelId = message["authorChannelId"];
|
|
addChannelToTreat(threadId, channelId);
|
|
}
|
|
}
|
|
json lastMessage = snippet.back();
|
|
// If there isn't any new message, then we stop the retrieving.
|
|
if(lastMessageTimestampRelativeMsec == lastMessage["videoOffsetTimeMsec"])
|
|
{
|
|
break;
|
|
}
|
|
lastMessageTimestampRelativeMsec = lastMessage["videoOffsetTimeMsec"];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
videoIds.clear();
|
|
}
|
|
if(hasNextPageToken)
|
|
{
|
|
pageToken = data["nextPageToken"];
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// This function verifies that the given hasn't already been treated.
|
|
void addChannelToTreat(unsigned short threadId, string channelId)
|
|
{
|
|
channelsAlreadyTreatedAndToTreatMutex.lock();
|
|
if(channelsAlreadyTreated.find(channelId) == channelsAlreadyTreated.end() && channelsToTreatRev.find(channelId) == channelsToTreatRev.end())
|
|
{
|
|
unsigned int channelsToTreatIndex = channelsToTreat.end()->first + 1;
|
|
channelsToTreat[channelsToTreatIndex] = channelId;
|
|
channelsToTreatRev[channelId] = channelsToTreatIndex;
|
|
|
|
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
|
|
|
writeFile(threadId, CHANNELS_FILE_PATH, "a", "\n" + channelId);
|
|
}
|
|
else
|
|
{
|
|
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
|
}
|
|
}
|
|
|
|
void treatComment(unsigned short threadId, json comment, string channelId)
|
|
{
|
|
json snippet = comment["snippet"];
|
|
// The `else` case can happen (cf `95a9421ad0469a09335afeddb2983e31dc00bc36`).
|
|
if(snippet.contains("authorChannelId"))
|
|
{
|
|
string channelId = snippet["authorChannelId"]["value"];
|
|
addChannelToTreat(threadId, channelId);
|
|
}
|
|
commentsCountThreads[threadId]++;
|
|
commentsPerSecondCount++;
|
|
}
|
|
|
|
string join(vector<string> parts, string delimiter)
|
|
{
|
|
string result = "";
|
|
unsigned int partsSize = parts.size();
|
|
for(unsigned int partsIndex = 0; partsIndex < partsSize; partsIndex++)
|
|
{
|
|
result += parts[partsIndex];
|
|
if(partsIndex < partsSize - 1)
|
|
{
|
|
result += delimiter;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
string exec(string cmd)
|
|
{
|
|
array<char, 128> buffer;
|
|
string result;
|
|
unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
|
|
if (!pipe)
|
|
{
|
|
throw runtime_error("popen() failed!");
|
|
}
|
|
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
|
|
{
|
|
result += buffer.data();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool writeFile(unsigned short threadId, string filePath, string option, string toWrite)
|
|
{
|
|
FILE* file = fopen(filePath.c_str(), option.c_str());
|
|
if(file != NULL)
|
|
{
|
|
fputs(toWrite.c_str(), file);
|
|
fclose(file);
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
PRINT("writeFile error: " << strerror(errno))
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool doesFileExist(string filePath)
|
|
{
|
|
struct stat buffer;
|
|
return stat(filePath.c_str(), &buffer) == 0;
|
|
}
|
|
|
|
void createDirectory(string path)
|
|
{
|
|
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
|
|
}
|
|
|
|
void deleteDirectory(string path)
|
|
{
|
|
filesystem::remove_all(path);
|
|
}
|
|
|
|
string getDate()
|
|
{
|
|
auto t = time(nullptr);
|
|
auto tm = *localtime(&t);
|
|
ostringstream toString;
|
|
toString << put_time(&tm, "%d-%m-%Y %H-%M-%S.");
|
|
milliseconds ms = duration_cast<milliseconds>(
|
|
system_clock::now().time_since_epoch()
|
|
);
|
|
toString << (ms.count() % 1000);
|
|
return toString.str();
|
|
}
|
|
|
|
set<string> setFromVector(vector<string> vec)
|
|
{
|
|
return set(vec.begin(), vec.end());
|
|
}
|
|
|
|
vector<string> getFileContent(string filePath)
|
|
{
|
|
vector<string> lines;
|
|
ifstream infile(filePath.c_str());
|
|
string line;
|
|
while(getline(infile, line))
|
|
lines.push_back(line);
|
|
return lines;
|
|
}
|
|
|
|
json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, string directoryPath, getJsonBehavior behavior)
|
|
{
|
|
string finalUrl = usingYoutubeDataApiv3 ?
|
|
(USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE ?
|
|
"https://yt.lemnoslife.com/noKey/" + url :
|
|
"https://www.googleapis.com/youtube/v3/" + url + "&key=" + apiKey) :
|
|
YOUTUBE_OPERATIONAL_API_INSTANCE_URL + "/" + url,
|
|
content = getHttps(finalUrl);
|
|
json data;
|
|
try
|
|
{
|
|
data = json::parse(content);
|
|
}
|
|
catch (json::parse_error& ex)
|
|
{
|
|
EXIT_WITH_ERROR("Parse error for " << finalUrl << ", as got: " << content << " !")
|
|
}
|
|
|
|
if(data.contains("error"))
|
|
{
|
|
string reason = data["error"]["errors"][0]["reason"];
|
|
// Contrarily to YouTube operational API no-key service we don't rotate keys in `KEYS_FILE_PATH`, as we keep them in memory here.
|
|
if(reason == "quotaExceeded")
|
|
{
|
|
quotaMutex.lock();
|
|
keys.erase(keys.begin());
|
|
keys.push_back(apiKey);
|
|
PRINT("No more quota on " << apiKey << " switching to " << keys[0] << ".")
|
|
apiKey = keys[0];
|
|
quotaMutex.unlock();
|
|
return getJson(threadId, url, true, directoryPath);
|
|
}
|
|
PRINT("Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
|
|
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
|
|
{
|
|
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath);
|
|
}
|
|
}
|
|
|
|
ostringstream toString;
|
|
toString << CHANNELS_DIRECTORY << directoryPath << "/" << requestsPerChannelThreads[threadId] << ".json";
|
|
requestsPerChannelThreads[threadId]++;
|
|
writeFile(threadId, toString.str(), "w", url + "\n" + content);
|
|
|
|
return data;
|
|
}
|
|
|
|
void print(ostringstream* toPrint)
|
|
{
|
|
printMutex.lock();
|
|
|
|
cout << getDate() << ": " << toPrint->str() << endl;
|
|
toPrint->str("");
|
|
|
|
printMutex.unlock();
|
|
}
|
|
|
|
// Is this function really multi-threading friendly? If not, could consider executing `curl` using the command line.
|
|
string getHttps(string url)
|
|
{
|
|
CURL* curl = curl_easy_init();
|
|
string got;
|
|
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
|
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1);
|
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
|
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &got);
|
|
curl_easy_perform(curl);
|
|
curl_easy_cleanup(curl);
|
|
return got;
|
|
}
|
|
|
|
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp)
|
|
{
|
|
((string*)userp)->append((char*)contents, size * nmemb);
|
|
return size * nmemb;
|
|
}
|