YouTube_captions_search_engine/main.cpp

#include <iostream>
#include <fstream>
#include <set>
#include <sys/stat.h>
#include <curl/curl.h>
#include <nlohmann/json.hpp>
using namespace std;
using json = nlohmann::json;

vector<string> getFileContent(string filePath);
json getJson(string url);
void createDirectory(string path),
     print(ostringstream* toPrint),
     treatComment(json comment);
string getHttps(string url);
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath),
     writeFile(string filePath, string option, string toWrite);

#define API_KEY "AIzaSy..."

// Note that this printing approach is only safe in a mono-thread context.
#define PRINT(x) toPrint << x; print(&toPrint);
ostringstream toPrint;

set<string> channelsAlreadyTreated,
            channelsToTreat;
unsigned int commentsCount = 0;

int main()
{
    string channelsToTreatFilePath = "channelsToTreat.txt";
    vector<string> channelsToTreatVec = getFileContent(channelsToTreatFilePath);
    channelsToTreat = set(channelsToTreatVec.begin(), channelsToTreatVec.end());

    string channelsDirectory = "channels/";
    createDirectory(channelsDirectory);

    for(const auto& entry : filesystem::directory_iterator(channelsDirectory))
    {
        channelsAlreadyTreated.insert(entry.path().filename());
    }

    PRINT(channelsToTreat.size() << " channel(s) to treat")
    PRINT(channelsAlreadyTreated.size() << " channel(s) already treated")

    while(!channelsToTreat.empty())
    {
        string channelToTreat = *channelsToTreat.begin();
        PRINT("Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")

        string pageToken = "";
        while(true)
        {
            json data = getJson("commentThreads?part=snippet,replies&allThreadsRelatedToChannelId=" + channelToTreat + "&maxResults=100&pageToken=" + pageToken);
            bool doesRelyingOnCommentThreadsIsEnough = data["error"]["errors"][0]["reason"] != "commentsDisabled";
            if(doesRelyingOnCommentThreadsIsEnough)
            {
                json items = data["items"];
                for(const auto& item : items)
                {
                    json comment = item["snippet"]["topLevelComment"];
                    string commentId = comment["id"];
                    treatComment(comment);
                    if(item.contains("replies"))
                    {
                        json replies = item["replies"]["comments"];
                        if(replies.size() >= 5)
                        {
                            string pageToken = "";
                            while(true)
                            {
                                json data = getJson("comments?part=snippet&parentId=" + commentId + "&maxResults=100&pageToken=" + pageToken);
                                json items = data["items"];
                                for(const auto& item : items)
                                {
                                    treatComment(item);
                                }
                                if(data.contains("nextPageToken"))
                                {
                                    pageToken = data["nextPageToken"];
                                }
                                else
                                {
                                    break;
                                }
                            }
                        }
                        else
                        {
                            for(const auto& reply : replies)
                            {
                                treatComment(reply);
                            }
                        }
                    }
                }
                if(data.contains("nextPageToken"))
                {
                    pageToken = data["nextPageToken"];
                }
                else
                {
                    break;
                }
            }
            else
            {
                PRINT("Comments disabled channel!")
                exit(1);
            }
        }

        PRINT(commentsCount)
        commentsCount = 0;

        channelsToTreat.erase(channelToTreat);
        channelsAlreadyTreated.insert(channelToTreat);

        string channelToTreatDirectory = channelsDirectory + channelToTreat + "/";
        createDirectory(channelToTreatDirectory);

        string toWrite = (doesFileExist(channelsToTreatFilePath) ? "\n" : "") + channelToTreat;
        writeFile(channelsToTreatFilePath, "a", toWrite);
    }

    return 0;
}

void treatComment(json comment)
{
    json snippet = comment["snippet"];
    // The `else` case can happen (cf `95a9421ad0469a09335afeddb2983e31dc00bc36`).
    if(snippet.contains("authorChannelId"))
    {
        string channelId = snippet["authorChannelId"]["value"];
        if(find(channelsAlreadyTreated.begin(), channelsAlreadyTreated.end(), channelId) == channelsAlreadyTreated.end())
            channelsToTreat.insert(channelId);
    }
    commentsCount++;
}

bool writeFile(string filePath, string option, string toWrite)
{
    FILE* file = fopen(filePath.c_str(), option.c_str());
    if(file != NULL)
    {
        fputs(toWrite.c_str(), file);
        fclose(file);
        return true;
    }
    return false;
}

bool doesFileExist(string filePath)
{
    struct stat buffer;
    return stat(filePath.c_str(), &buffer) == 0;
}

void createDirectory(string path)
{
    mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
}

string getDate()
{
    auto t = time(nullptr);
    auto tm = *localtime(&t);
    ostringstream toString;
    toString << put_time(&tm, "%d-%m-%Y %H-%M-%S");
    return toString.str();
}

vector<string> getFileContent(string filePath)
{
    vector<string> lines;
    ifstream infile(filePath.c_str());
    string line;
    while(getline(infile, line))
        lines.push_back(line);
    return lines;
}

json getJson(string url)
{
    url = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + API_KEY;
    string content = getHttps(url);
    json data = json::parse(content);
    return data;
}

void print(ostringstream* toPrint)
{
    cout << getDate() << ": " << toPrint->str() << endl;
    toPrint->str("");
}

string getHttps(string url)
{
    CURL* curl = curl_easy_init();
    string got;
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &got);
    curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    return got;
}

size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
    ((string*)userp)->append((char*)contents, size * nmemb);
    return size * nmemb;
}