Fix #6: Add support for multiple keys to be resilient against exceeded quota errors
This commit is contained in:
parent
d6f6b26361
commit
eb805f5ced
35
main.cpp
35
main.cpp
@ -14,6 +14,7 @@ using json = nlohmann::json;
|
|||||||
|
|
||||||
enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNotFound };
|
enum getJsonBehavior { normal, retryOnCommentsDisabled, returnErrorIfPlaylistNotFound };
|
||||||
|
|
||||||
|
set<string> setFromVector(vector<string> vec);
|
||||||
vector<string> getFileContent(string filePath);
|
vector<string> getFileContent(string filePath);
|
||||||
json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior = normal);
|
json getJson(unsigned short threadId, string url, string directoryPath, getJsonBehavior behavior = normal);
|
||||||
void createDirectory(string path),
|
void createDirectory(string path),
|
||||||
@ -29,21 +30,24 @@ bool doesFileExist(string filePath),
|
|||||||
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
||||||
|
|
||||||
#define USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
|
#define USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
|
||||||
#define API_KEY "AIzaSy..."
|
|
||||||
#define THREADS_NUMBER 10
|
#define THREADS_NUMBER 10
|
||||||
|
|
||||||
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
|
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
|
||||||
#define DEFAULT_THREAD_ID 0
|
#define DEFAULT_THREAD_ID 0
|
||||||
|
|
||||||
mutex printMutex,
|
mutex printMutex,
|
||||||
channelsAlreadyTreatedAndToTreatMutex;
|
channelsAlreadyTreatedAndToTreatMutex,
|
||||||
|
quotaMutex;
|
||||||
set<string> channelsAlreadyTreated,
|
set<string> channelsAlreadyTreated,
|
||||||
channelsToTreat;
|
channelsToTreat;
|
||||||
|
vector<string> keys;
|
||||||
unsigned int commentsCount = 0,
|
unsigned int commentsCount = 0,
|
||||||
commentsPerSecondCount = 0,
|
commentsPerSecondCount = 0,
|
||||||
requestsPerChannel = 0;
|
requestsPerChannel = 0;
|
||||||
string CHANNELS_DIRECTORY = "channels/",
|
string CHANNELS_DIRECTORY = "channels/",
|
||||||
CHANNELS_FILE_PATH = "channels.txt";
|
CHANNELS_FILE_PATH = "channels.txt",
|
||||||
|
KEYS_FILE_PATH = "keys.txt",
|
||||||
|
apiKey = ""; // Will firstly be filled with `KEYS_FILE_PATH` first line.
|
||||||
|
|
||||||
int main()
|
int main()
|
||||||
{
|
{
|
||||||
@ -52,7 +56,10 @@ int main()
|
|||||||
// On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set.
|
// On a restart, `CHANNELS_FILE_PATH` is read and every channel not found in `CHANNELS_DIRECTORY` is added to `channelsToTreat` or `channelsToTreat` otherwise before continuing, as if `CHANNELS_FILE_PATH` was containing a **treated** starting set.
|
||||||
vector<string> channelsVec = getFileContent(CHANNELS_FILE_PATH);
|
vector<string> channelsVec = getFileContent(CHANNELS_FILE_PATH);
|
||||||
// Note that using `set`s makes the search faster but we lose the `channels.txt` lines order.
|
// Note that using `set`s makes the search faster but we lose the `channels.txt` lines order.
|
||||||
channelsToTreat = set(channelsVec.begin(), channelsVec.end());
|
channelsToTreat = setFromVector(channelsVec);
|
||||||
|
|
||||||
|
keys = getFileContent(KEYS_FILE_PATH);
|
||||||
|
apiKey = keys[0];
|
||||||
|
|
||||||
createDirectory(CHANNELS_DIRECTORY);
|
createDirectory(CHANNELS_DIRECTORY);
|
||||||
|
|
||||||
@ -332,6 +339,11 @@ string getDate()
|
|||||||
return toString.str();
|
return toString.str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
set<string> setFromVector(vector<string> vec)
|
||||||
|
{
|
||||||
|
return set(vec.begin(), vec.end());
|
||||||
|
}
|
||||||
|
|
||||||
vector<string> getFileContent(string filePath)
|
vector<string> getFileContent(string filePath)
|
||||||
{
|
{
|
||||||
vector<string> lines;
|
vector<string> lines;
|
||||||
@ -347,7 +359,7 @@ json getJson(unsigned short threadId, string url, string directoryPath, getJsonB
|
|||||||
#ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
|
#ifdef USE_YT_LEMNOSLIFE_COM_NO_KEY_SERVICE
|
||||||
string finalUrl = "https://yt.lemnoslife.com/noKey/" + url;
|
string finalUrl = "https://yt.lemnoslife.com/noKey/" + url;
|
||||||
#else
|
#else
|
||||||
string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + API_KEY;
|
string finalUrl = "https://www.googleapis.com/youtube/v3/" + url + "&key=" + apiKey;
|
||||||
#endif
|
#endif
|
||||||
string content = getHttps(finalUrl);
|
string content = getHttps(finalUrl);
|
||||||
json data;
|
json data;
|
||||||
@ -363,8 +375,19 @@ json getJson(unsigned short threadId, string url, string directoryPath, getJsonB
|
|||||||
|
|
||||||
if(data.contains("error"))
|
if(data.contains("error"))
|
||||||
{
|
{
|
||||||
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
|
|
||||||
string reason = data["error"]["errors"][0]["reason"];
|
string reason = data["error"]["errors"][0]["reason"];
|
||||||
|
// Contrarily to YouTube operational API no-key service we don't rotate keys in `KEYS_FILE_PATH`, as we keep them in memory here.
|
||||||
|
if(reason == "quotaExceeded")
|
||||||
|
{
|
||||||
|
quotaMutex.lock();
|
||||||
|
keys.erase(keys.begin());
|
||||||
|
keys.push_back(apiKey);
|
||||||
|
PRINT(threadId, "No more quota on " << apiKey << " switching to " << keys[0] << ".")
|
||||||
|
apiKey = keys[0];
|
||||||
|
quotaMutex.unlock();
|
||||||
|
return getJson(threadId, url, directoryPath);
|
||||||
|
}
|
||||||
|
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
|
||||||
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
|
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
|
||||||
{
|
{
|
||||||
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, directoryPath);
|
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, directoryPath);
|
||||||
|
Loading…
Reference in New Issue
Block a user