Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Fix #24: Stop using macros for user inputs to notably make releases

Browse Source
This commit is contained in:
Benjamin Loison 2023-01-08 18:26:20 +01:00
parent eb805f5ced
commit 27cd5c3a64
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8
1 changed files with 35 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
main.cpp
View File

@ -29,11 +29,9 @@ size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath), bool doesFileExist(string filePath),
writeFile(unsigned short threadId, string filePath, string option, string toWrite); writeFile(unsigned short threadId, string filePath, string option, string toWrite);
#define USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
#define THREADS_NUMBER 10
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); } #define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
#define DEFAULT_THREAD_ID 0 #define DEFAULT_THREAD_ID 0
#define MAIN_PRINT(x) PRINT(DEFAULT_THREAD_ID, x)
mutex printMutex, mutex printMutex,
channelsAlreadyTreatedAndToTreatMutex, channelsAlreadyTreatedAndToTreatMutex,
@ -44,13 +42,38 @@ vector<string> keys;
unsigned int commentsCount = 0, unsigned int commentsCount = 0,
commentsPerSecondCount = 0, commentsPerSecondCount = 0,
requestsPerChannel = 0; requestsPerChannel = 0;
unsigned short THREADS_NUMBER = 1;
string CHANNELS_DIRECTORY = "channels/", string CHANNELS_DIRECTORY = "channels/",
CHANNELS_FILE_PATH = "channels.txt", CHANNELS_FILE_PATH = "channels.txt",
KEYS_FILE_PATH = "keys.txt", KEYS_FILE_PATH = "keys.txt",
apiKey = ""; // Will firstly be filled with `KEYS_FILE_PATH` first line. apiKey = ""; // Will firstly be filled with `KEYS_FILE_PATH` first line.
bool USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE = false;
int main() int main(int argc, char *argv[])
{ {
for(unsigned short argvIndex = 1; argvIndex < argc; argvIndex++)
{
string argvStr = string(argv[argvIndex]);
if(argvStr == "--no-keys")
{
USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE = true;
}
else if(argvStr.rfind("--threads=", 0) == 0)
{
THREADS_NUMBER = atoi(argvStr.substr(10).c_str());
}
else if(argvStr == "-h" || argvStr == "--help")
{
MAIN_PRINT("Usage: " << argv[0] << " [--help/-h] [--no-keys] [--threads=N]")
exit(0);
}
else
{
MAIN_PRINT("Unrecognized parameter " << argvStr)
exit(1);
}
}
// The starting set should be written to `CHANNELS_FILE_PATH`. // The starting set should be written to `CHANNELS_FILE_PATH`.
// To resume this algorithm after a shutdown, just restart it after having deleted the last channel folders in `CHANNELS_DIRECTORY` being treated. // To resume this algorithm after a shutdown, just restart it after having deleted the last channel folders in `CHANNELS_DIRECTORY` being treated.
// On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set. // On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set.
@ -71,18 +94,18 @@ int main()
channelsAlreadyTreated.insert(channelId); channelsAlreadyTreated.insert(channelId);
} }
PRINT(DEFAULT_THREAD_ID, channelsToTreat.size() << " channel(s) to treat") MAIN_PRINT(channelsToTreat.size() << " channel(s) to treat")
PRINT(DEFAULT_THREAD_ID, channelsAlreadyTreated.size() << " channel(s) already treated") MAIN_PRINT(channelsAlreadyTreated.size() << " channel(s) already treated")
thread threads[THREADS_NUMBER]; vector<thread> threads;
for(unsigned short threadsIndex = 0; threadsIndex < THREADS_NUMBER; threadsIndex++) for(unsigned short threadsIndex = 0; threadsIndex < THREADS_NUMBER; threadsIndex++)
{ {
threads[threadsIndex] = thread(treatChannels, threadsIndex + 1); threads.push_back(thread(treatChannels, threadsIndex + 1));
} }
while(true) while(true)
{ {
PRINT(DEFAULT_THREAD_ID, "Comments per second: " << commentsPerSecondCount) MAIN_PRINT("Comments per second: " << commentsPerSecondCount)
commentsPerSecondCount = 0; commentsPerSecondCount = 0;
sleep(1); sleep(1);
} }
@ -356,12 +379,9 @@ vector<string> getFileContent(string filePath)
json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior) json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior)
{ {
#ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE string finalUrl = USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE ? "https://yt.lemnoslife.com/noKey/" + url :
string finalUrl = "https://yt.lemnoslife.com/noKey/" + url; "https://www.googleapis.com/youtube/v3/" + url + "&key=" + apiKey,
#else content = getHttps(finalUrl);
string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + apiKey;
#endif
string content = getHttps(finalUrl);
json data; json data;
try try
{ {