Benjamin_Loison/YouTube_captions_search_engine
Benjamin_Loison/YouTube_captions_search_engine
1
0
Fork 0
Code Issues 16 Pull Requests Packages Projects Releases 2 Wiki Activity

Fix #6: Add support for multiple keys to be resilient against exceeded quota errors

Browse Source
This commit is contained in:
Benjamin Loison 2023-01-08 17:59:08 +01:00
parent 1ee767abbc
commit d498c86058
2 changed files with 31 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
keys.txt Normal file
View File

@ -0,0 +1,2 @@
AIzaSy...
AIzaSy...

35
main.cpp
View File

@ -14,6 +14,7 @@ using json = nlohmann::json;
enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNotFound }; enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNotFound };
set<string> setFromVector(vector<string> vec);
vector<string> getFileContent(string filePath); vector<string> getFileContent(string filePath);
json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior = normal); json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior = normal);
void createDirectory(string path), void createDirectory(string path),
@ -29,21 +30,24 @@ bool doesFileExist(string filePath),
writeFile(unsigned short threadId, string filePath, string option, string toWrite); writeFile(unsigned short threadId, string filePath, string option, string toWrite);
#define USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE #define USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
#define API_KEY "AIzaSy..."
#define THREADS_NUMBER 10 #define THREADS_NUMBER 10
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); } #define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
#define DEFAULT_THREAD_ID 0 #define DEFAULT_THREAD_ID 0
mutex printMutex, mutex printMutex,
channelsAlreadyTreatedAndToTreatMutex; channelsAlreadyTreatedAndToTreatMutex,
quotaMutex;
set<string> channelsAlreadyTreated, set<string> channelsAlreadyTreated,
channelsToTreat; channelsToTreat;
vector<string> keys;
unsigned int commentsCount = 0, unsigned int commentsCount = 0,
commentsPerSecondCount = 0, commentsPerSecondCount = 0,
requestsPerChannel = 0; requestsPerChannel = 0;
string CHANNELS_DIRECTORY = "channels/", string CHANNELS_DIRECTORY = "channels/",
CHANNELS_FILE_PATH = "channels.txt"; CHANNELS_FILE_PATH = "channels.txt",
KEYS_FILE_PATH = "keys.txt",
apiKey = ""; // Will firstly be filled with `KEYS_FILE_PATH` first line.
int main() int main()
{ {
@ -52,7 +56,10 @@ int main()
// On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set. // On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set.
vector<string> channelsVec = getFileContent(CHANNELS_FILE_PATH); vector<string> channelsVec = getFileContent(CHANNELS_FILE_PATH);
// Note that using `set`s makes the search faster but we lose the `channels.txt` lines order. // Note that using `set`s makes the search faster but we lose the `channels.txt` lines order.
channelsToTreat = set(channelsVec.begin(), channelsVec.end()); channelsToTreat = setFromVector(channelsVec);
keys = getFileContent(KEYS_FILE_PATH);
apiKey = keys[0];
createDirectory(CHANNELS_DIRECTORY); createDirectory(CHANNELS_DIRECTORY);
@ -332,6 +339,11 @@ string getDate()
return toString.str(); return toString.str();
} }
set<string> setFromVector(vector<string> vec)
{
return set(vec.begin(), vec.end());
}
vector<string> getFileContent(string filePath) vector<string> getFileContent(string filePath)
{ {
vector<string> lines; vector<string> lines;
@ -347,7 +359,7 @@ json getJson(unsigned short threadId, string url, string directoryPath, getJsonB
#ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE #ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
string finalUrl = "https://yt.lemnoslife.com/noKey/" + url; string finalUrl = "https://yt.lemnoslife.com/noKey/" + url;
#else #else
string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + API_KEY; string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + apiKey;
#endif #endif
string content = getHttps(finalUrl); string content = getHttps(finalUrl);
json data; json data;
@ -363,8 +375,19 @@ json getJson(unsigned short threadId, string url, string directoryPath, getJsonB
if(data.contains("error")) if(data.contains("error"))
{ {
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
string reason = data["error"]["errors"][0]["reason"]; string reason = data["error"]["errors"][0]["reason"];
// Contrarily to YouTube operational API no-key service we don't rotate keys in `KEYS_FILE_PATH`, as we keep them in memory here.
if(reason == "quotaExceeded")
{
quotaMutex.lock();
keys.erase(keys.begin());
keys.push_back(apiKey);
PRINT(threadId, "No more quota on " << apiKey << " switching to " << keys[0] << ".")
apiKey = keys[0];
quotaMutex.unlock();
return getJson(threadId, url, directoryPath);
}
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled) if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
{ {
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, directoryPath); return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, directoryPath);