YouTube_captions_search_engine/main.cpp

218 lines
6.8 KiB
C++
Raw Normal View History

#include <iostream>
#include <fstream>
#include <set>
#include <sys/stat.h>
#include <curl/curl.h>
#include <nlohmann/json.hpp>
using namespace std;
using json = nlohmann::json;
vector<string> getFileContent(string filePath);
json getJson(string url);
void createDirectory(string path),
print(ostringstream* toPrint),
treatComment(json comment);
string getHttps(string url);
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath),
writeFile(string filePath, string option, string toWrite);
#define API_KEY "AIzaSy..."
// Note that this printing approach is only safe in a mono-thread context.
#define PRINT(x) toPrint << x; print(&toPrint);
ostringstream toPrint;
set<string> channelsAlreadyTreated,
channelsToTreat;
unsigned int commentsCount = 0;
int main()
{
string channelsToTreatFilePath = "channelsToTreat.txt";
vector<string> channelsToTreatVec = getFileContent(channelsToTreatFilePath);
channelsToTreat = set(channelsToTreatVec.begin(), channelsToTreatVec.end());
string channelsDirectory = "channels/";
createDirectory(channelsDirectory);
for(const auto& entry : filesystem::directory_iterator(channelsDirectory))
{
channelsAlreadyTreated.insert(entry.path().filename());
}
PRINT(channelsToTreat.size() << " channel(s) to treat")
PRINT(channelsAlreadyTreated.size() << " channel(s) already treated")
while(!channelsToTreat.empty())
{
string channelToTreat = *channelsToTreat.begin();
PRINT("Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
string pageToken = "";
while(true)
{
json data = getJson("commentThreads?part=snippet,replies&allThreadsRelatedToChannelId=" + channelToTreat + "&maxResults=100&pageToken=" + pageToken);
bool doesRelyingOnCommentThreadsIsEnough = data["error"]["errors"][0]["reason"] != "commentsDisabled";
if(doesRelyingOnCommentThreadsIsEnough)
{
json items = data["items"];
for(const auto& item : items)
{
json comment = item["snippet"]["topLevelComment"];
string commentId = comment["id"];
treatComment(comment);
if(item.contains("replies"))
{
json replies = item["replies"]["comments"];
if(replies.size() >= 5)
{
string pageToken = "";
while(true)
{
json data = getJson("comments?part=snippet&parentId=" + commentId + "&maxResults=100&pageToken=" + pageToken);
json items = data["items"];
for(const auto& item : items)
{
treatComment(item);
}
if(data.contains("nextPageToken"))
{
pageToken = data["nextPageToken"];
}
else
{
break;
}
}
}
else
{
for(const auto& reply : replies)
{
treatComment(reply);
}
}
}
}
if(data.contains("nextPageToken"))
{
pageToken = data["nextPageToken"];
}
else
{
break;
}
}
else
{
PRINT("Comments disabled channel!")
exit(1);
}
}
PRINT(commentsCount)
commentsCount = 0;
channelsToTreat.erase(channelToTreat);
channelsAlreadyTreated.insert(channelToTreat);
string channelToTreatDirectory = channelsDirectory + channelToTreat + "/";
createDirectory(channelToTreatDirectory);
string toWrite = (doesFileExist(channelsToTreatFilePath) ? "\n" : "") + channelToTreat;
writeFile(channelsToTreatFilePath, "a", toWrite);
}
return 0;
}
void treatComment(json comment)
{
json snippet = comment["snippet"];
// The `else` case can happen (cf `95a9421ad0469a09335afeddb2983e31dc00bc36`).
if(snippet.contains("authorChannelId"))
{
string channelId = snippet["authorChannelId"]["value"];
if(find(channelsAlreadyTreated.begin(), channelsAlreadyTreated.end(), channelId) == channelsAlreadyTreated.end())
channelsToTreat.insert(channelId);
}
commentsCount++;
}
bool writeFile(string filePath, string option, string toWrite)
{
FILE* file = fopen(filePath.c_str(), option.c_str());
if(file != NULL)
{
fputs(toWrite.c_str(), file);
fclose(file);
return true;
}
return false;
}
bool doesFileExist(string filePath)
{
struct stat buffer;
return stat(filePath.c_str(), &buffer) == 0;
}
void createDirectory(string path)
{
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
}
2022-12-22 05:47:16 +01:00
string getDate()
{
auto t = time(nullptr);
auto tm = *localtime(&t);
ostringstream toString;
toString << put_time(&tm, "%d-%m-%Y %H-%M-%S");
return toString.str();
}
vector<string> getFileContent(string filePath)
{
vector<string> lines;
ifstream infile(filePath.c_str());
string line;
while(getline(infile, line))
lines.push_back(line);
return lines;
}
json getJson(string url)
{
url = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + API_KEY;
string content = getHttps(url);
json data = json::parse(content);
return data;
}
void print(ostringstream* toPrint)
{
cout << getDate() << ": " << toPrint->str() << endl;
toPrint->str("");
}
string getHttps(string url)
{
CURL* curl = curl_easy_init();
string got;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &got);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
return got;
}
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
((string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}