Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Add logging to exec and make it crashless, requests and captions folders support for compressing, clean captions support for videos being livestreams and videos starting with -

Browse Source
This commit is contained in:
Benjamin Loison 2023-02-12 16:24:16 +01:00
parent 8cf5698051
commit 54fe40e588
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
main.cpp
View File

@ -23,9 +23,9 @@ void createDirectory(string path),
treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat), treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat),
treatChannels(unsigned short threadId), treatChannels(unsigned short threadId),
deleteDirectory(string path), deleteDirectory(string path),
addChannelToTreat(unsigned short threadId, string channelId); addChannelToTreat(unsigned short threadId, string channelId),
exec(unsigned short threadId, string cmd, bool debug = true);
string getHttps(string url), string getHttps(string url),
exec(unsigned short threadId, string cmd),
join(vector<string> parts, string delimiter); join(vector<string> parts, string delimiter);
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp); size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath), bool doesFileExist(string filePath),
@ -189,7 +189,9 @@ void treatChannels(unsigned short threadId)
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds. // Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
PRINT("Starting compression...") PRINT("Starting compression...")
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli. // As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
exec(threadId, "cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@"); // We precise no `debug`ging, as otherwise the zipping operation doesn't work as expected.
// As the zipping process isn't recursive, we can't just rely on `ls`, but we are obliged to use `find`.
exec(threadId, "cd " + channelToTreatDirectory + " && find | zip ../" + channelToTreat + ".zip -@", false);
PRINT("Compression finished, started deleting initial directory...") PRINT("Compression finished, started deleting initial directory...")
deleteDirectory(channelToTreatDirectory); deleteDirectory(channelToTreatDirectory);
@ -586,9 +588,10 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
// Firstly download all not automatically generated captions. // Firstly download all not automatically generated captions.
// The underscore in `-o` argument is used to not end up with hidden files. // The underscore in `-o` argument is used to not end up with hidden files.
// We are obliged to precise the video id after `--`, otherwise if the video id starts with `-` it's considered as an argument.
string cmdCommonPrefix = "yt-dlp --skip-download ", string cmdCommonPrefix = "yt-dlp --skip-download ",
cmdCommonPostfix = " '" + videoId + "' -o '" + channelCaptionsToTreatDirectory + "_'"; cmdCommonPostfix = " -o '" + channelCaptionsToTreatDirectory + "_' -- " + videoId;
string cmd = cmdCommonPrefix + "--all-subs" + cmdCommonPostfix; string cmd = cmdCommonPrefix + "--sub-lang all,-live_chat" + cmdCommonPostfix;
exec(threadId, cmd); exec(threadId, cmd);
// Secondly download the automatically generated captions. // Secondly download the automatically generated captions.
@ -655,25 +658,22 @@ string join(vector<string> parts, string delimiter)
return result; return result;
} }
string exec(unsigned short threadId, string cmd) void exec(unsigned short threadId, string cmd, bool debug)
{ {
if(debug)
{
ostringstream toString; ostringstream toString;
toString << threadId; toString << threadId;
string threadIdStr = toString.str(), debugCommonFilePath = DEBUG_DIRECTORY + threadIdStr; string initialCmd = cmd,
cmd += " >> " + debugCommonFilePath + ".out"; threadIdStr = toString.str(),
cmd += " 2>> " + debugCommonFilePath + ".err"; debugCommonFilePath = DEBUG_DIRECTORY + threadIdStr,
array<char, 128> buffer; debugOutFilePath = debugCommonFilePath + ".out",
string result; debugErrFilePath = debugCommonFilePath + ".err";
unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose); cmd += " >> " + debugOutFilePath;
if (!pipe) cmd += " 2>> " + debugErrFilePath;
{ cmd += "; echo \"" + initialCmd + "\" | tee -a " + debugOutFilePath + " " + debugErrFilePath;
throw runtime_error("popen() failed!");
} }
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) system(cmd.c_str());
{
result += buffer.data();
}
return result;
} }
bool writeFile(unsigned short threadId, string filePath, string option, string toWrite) bool writeFile(unsigned short threadId, string filePath, string option, string toWrite)