diff --git a/keys.txt b/keys.txt new file mode 100644 index 0000000..da4c8cd --- /dev/null +++ b/keys.txt @@ -0,0 +1,2 @@ +AIzaSy... +AIzaSy... \ No newline at end of file diff --git a/main.cpp b/main.cpp index 4c22519..e350bfb 100644 --- a/main.cpp +++ b/main.cpp @@ -14,6 +14,7 @@ using json = nlohmann::json; enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNotFound }; +set setFromVector(vector vec); vector getFileContent(string filePath); json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior = normal); void createDirectory(string path), @@ -29,21 +30,24 @@ bool doesFileExist(string filePath), writeFile(unsigned short threadId, string filePath, string option, string toWrite); #define USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE -#define API_KEY "AIzaSy..." #define THREADS_NUMBER 10 #define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); } #define DEFAULT_THREAD_ID 0 mutex printMutex, - channelsAlreadyTreatedAndToTreatMutex; + channelsAlreadyTreatedAndToTreatMutex, + quotaMutex; set channelsAlreadyTreated, channelsToTreat; +vector keys; unsigned int commentsCount = 0, commentsPerSecondCount = 0, requestsPerChannel = 0; string CHANNELS_DIRECTORY = "channels/", - CHANNELS_FILE_PATH = "channels.txt"; + CHANNELS_FILE_PATH = "channels.txt", + KEYS_FILE_PATH = "keys.txt", + apiKey = ""; // Will firstly be filled with `KEYS_FILE_PATH` first line. int main() { @@ -52,7 +56,10 @@ int main() // On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set. vector channelsVec = getFileContent(CHANNELS_FILE_PATH); // Note that using `set`s makes the search faster but we lose the `channels.txt` lines order. - channelsToTreat = set(channelsVec.begin(), channelsVec.end()); + channelsToTreat = setFromVector(channelsVec); + + keys = getFileContent(KEYS_FILE_PATH); + apiKey = keys[0]; createDirectory(CHANNELS_DIRECTORY); @@ -332,6 +339,11 @@ string getDate() return toString.str(); } +set setFromVector(vector vec) +{ + return set(vec.begin(), vec.end()); +} + vector getFileContent(string filePath) { vector lines; @@ -347,7 +359,7 @@ json getJson(unsigned short threadId, string url, string directoryPath, getJsonB #ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE string finalUrl = "https://yt.lemnoslife.com/noKey/" + url; #else - string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + API_KEY; + string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + apiKey; #endif string content = getHttps(finalUrl); json data; @@ -363,8 +375,19 @@ json getJson(unsigned short threadId, string url, string directoryPath, getJsonB if(data.contains("error")) { - PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !") string reason = data["error"]["errors"][0]["reason"]; + // Contrarily to YouTube operational API no-key service we don't rotate keys in `KEYS_FILE_PATH`, as we keep them in memory here. + if(reason == "quotaExceeded") + { + quotaMutex.lock(); + keys.erase(keys.begin()); + keys.push_back(apiKey); + PRINT(threadId, "No more quota on " << apiKey << " switching to " << keys[0] << ".") + apiKey = keys[0]; + quotaMutex.unlock(); + return getJson(threadId, url, directoryPath); + } + PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !") if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled) { return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, directoryPath);