Make PRINT
not requiring to precise threadId
This commit is contained in:
parent
548a797ee8
commit
59dc5676cc
35
main.cpp
35
main.cpp
@ -30,9 +30,10 @@ size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
|
|||||||
bool doesFileExist(string filePath),
|
bool doesFileExist(string filePath),
|
||||||
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
||||||
|
|
||||||
#define PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
|
#define THREAD_PRINT(threadId, x) { ostringstream toPrint; toPrint << threadId << ": " << x; print(&toPrint); }
|
||||||
|
#define PRINT(x) THREAD_PRINT(threadId, x)
|
||||||
#define DEFAULT_THREAD_ID 0
|
#define DEFAULT_THREAD_ID 0
|
||||||
#define MAIN_PRINT(x) PRINT(DEFAULT_THREAD_ID, x)
|
#define MAIN_PRINT(x) THREAD_PRINT(DEFAULT_THREAD_ID, x)
|
||||||
|
|
||||||
mutex printMutex,
|
mutex printMutex,
|
||||||
channelsAlreadyTreatedAndToTreatMutex,
|
channelsAlreadyTreatedAndToTreatMutex,
|
||||||
@ -160,7 +161,7 @@ void treatChannels(unsigned short threadId)
|
|||||||
|
|
||||||
string channelToTreat = channelsToTreat.begin()->second;
|
string channelToTreat = channelsToTreat.begin()->second;
|
||||||
|
|
||||||
PRINT(threadId, "Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
|
PRINT("Treating channel " << channelToTreat << " (treated: " << channelsAlreadyTreated.size() << ", to treat: " << channelsToTreat.size() << ")")
|
||||||
|
|
||||||
commentsCountThreads[threadId] = 0;
|
commentsCountThreads[threadId] = 0;
|
||||||
requestsPerChannelThreads[threadId] = 0;
|
requestsPerChannelThreads[threadId] = 0;
|
||||||
@ -178,15 +179,15 @@ void treatChannels(unsigned short threadId)
|
|||||||
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
|
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
|
||||||
|
|
||||||
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
|
// Note that compressing the French most subscribers channel took 4 minutes and 42 seconds.
|
||||||
PRINT(threadId, "Starting compression...")
|
PRINT("Starting compression...")
|
||||||
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
|
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
|
||||||
exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@");
|
exec("cd " + channelToTreatDirectory + " && ls | zip ../" + channelToTreat + ".zip -@");
|
||||||
|
|
||||||
PRINT(threadId, "Compression finished, started deleting initial directory...")
|
PRINT("Compression finished, started deleting initial directory...")
|
||||||
deleteDirectory(channelToTreatDirectory);
|
deleteDirectory(channelToTreatDirectory);
|
||||||
PRINT(threadId, "Deleting directory finished.")
|
PRINT("Deleting directory finished.")
|
||||||
|
|
||||||
PRINT(threadId, commentsCountThreads[threadId] << " comments were found for this channel.")
|
PRINT(commentsCountThreads[threadId] << " comments were found for this channel.")
|
||||||
}
|
}
|
||||||
|
|
||||||
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
channelsAlreadyTreatedAndToTreatMutex.unlock();
|
||||||
@ -255,11 +256,11 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
PRINT(threadId, "Comments disabled channel, treating differently...")
|
PRINT("Comments disabled channel, treating differently...")
|
||||||
json data = getJson(threadId, "channels?part=statistics&id=" + channelToTreat, true, channelToTreat);
|
json data = getJson(threadId, "channels?part=statistics&id=" + channelToTreat, true, channelToTreat);
|
||||||
// YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer...
|
// YouTube Data API v3 Videos: list endpoint returns `videoCount` as a string and not an integer...
|
||||||
unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str());
|
unsigned int videoCount = atoi(string(data["items"][0]["statistics"]["videoCount"]).c_str());
|
||||||
PRINT(threadId, "The channel has about " << videoCount << " videos.")
|
PRINT("The channel has about " << videoCount << " videos.")
|
||||||
// `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here.
|
// `UC-3A9g4U1PpLaeAuD4jSP_w` has a `videoCount` of 2, while its `uploads` playlist contains 3 videos. So we use a strict inequality here.
|
||||||
if(0 < videoCount && videoCount < 20000)
|
if(0 < videoCount && videoCount < 20000)
|
||||||
{
|
{
|
||||||
@ -271,7 +272,7 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
|
|||||||
json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound);
|
json data = getJson(threadId, "playlistItems?part=snippet,contentDetails,status&playlistId=" + playlistToTreat + "&maxResults=50&pageToken=" + pageToken, true, channelToTreat, returnErrorIfPlaylistNotFound);
|
||||||
if(data.contains("error"))
|
if(data.contains("error"))
|
||||||
{
|
{
|
||||||
PRINT(threadId, "Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
|
PRINT("Not listing comments on videos, as `playlistItems` hasn't found the `uploads` playlist!")
|
||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
json items = data["items"];
|
json items = data["items"];
|
||||||
@ -295,12 +296,12 @@ void treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, str
|
|||||||
}
|
}
|
||||||
else if(videoCount == 0)
|
else if(videoCount == 0)
|
||||||
{
|
{
|
||||||
PRINT(threadId, "Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.")
|
PRINT("Skip listing comments on videos, as they shouldn't be any according to `channels?part=statistics`.")
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
else //if(videoCount >= 20000)
|
else //if(videoCount >= 20000)
|
||||||
{
|
{
|
||||||
PRINT(threadId, "The videos count of the channel exceeds the supported 20,000 limit!")
|
PRINT("The videos count of the channel exceeds the supported 20,000 limit!")
|
||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -437,7 +438,7 @@ breakChannelsTreatment:
|
|||||||
{
|
{
|
||||||
string videoId = snippet["resourceId"]["videoId"],
|
string videoId = snippet["resourceId"]["videoId"],
|
||||||
channelId = snippet["videoOwnerChannelId"];
|
channelId = snippet["videoOwnerChannelId"];
|
||||||
PRINT(threadId, "Found non public video (" << videoId << ") in: " << playlistId)
|
PRINT("Found non public video (" << videoId << ") in: " << playlistId)
|
||||||
string channelUnlistedVideosFilePath = CHANNELS_DIRECTORY + UNLISTED_VIDEOS_FILE_PATH;
|
string channelUnlistedVideosFilePath = CHANNELS_DIRECTORY + UNLISTED_VIDEOS_FILE_PATH;
|
||||||
bool doesChannelUnlistedVideosFileExist = doesFileExist(channelUnlistedVideosFilePath);
|
bool doesChannelUnlistedVideosFileExist = doesFileExist(channelUnlistedVideosFilePath);
|
||||||
writeFile(threadId, channelUnlistedVideosFilePath, !doesChannelUnlistedVideosFileExist ? "w" : "a", (!doesChannelUnlistedVideosFileExist ? "" : "\n") + channelId);
|
writeFile(threadId, channelUnlistedVideosFilePath, !doesChannelUnlistedVideosFileExist ? "w" : "a", (!doesChannelUnlistedVideosFileExist ? "" : "\n") + channelId);
|
||||||
@ -536,7 +537,7 @@ bool writeFile(unsigned short threadId, string filePath, string option, string t
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
PRINT(threadId, "writeFile error: " << strerror(errno))
|
PRINT("writeFile error: " << strerror(errno))
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -600,7 +601,7 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
|
|||||||
}
|
}
|
||||||
catch (json::parse_error& ex)
|
catch (json::parse_error& ex)
|
||||||
{
|
{
|
||||||
PRINT(threadId, "Parse error for " << finalUrl << ", as got: " << content << " !")
|
PRINT("Parse error for " << finalUrl << ", as got: " << content << " !")
|
||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -613,12 +614,12 @@ json getJson(unsigned short threadId, string url, bool usingYoutubeDataApiv3, st
|
|||||||
quotaMutex.lock();
|
quotaMutex.lock();
|
||||||
keys.erase(keys.begin());
|
keys.erase(keys.begin());
|
||||||
keys.push_back(apiKey);
|
keys.push_back(apiKey);
|
||||||
PRINT(threadId, "No more quota on " << apiKey << " switching to " << keys[0] << ".")
|
PRINT("No more quota on " << apiKey << " switching to " << keys[0] << ".")
|
||||||
apiKey = keys[0];
|
apiKey = keys[0];
|
||||||
quotaMutex.unlock();
|
quotaMutex.unlock();
|
||||||
return getJson(threadId, url, true, directoryPath);
|
return getJson(threadId, url, true, directoryPath);
|
||||||
}
|
}
|
||||||
PRINT(threadId, "Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
|
PRINT("Found error in JSON at URL: " << finalUrl << " for content: " << content << " !")
|
||||||
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
|
if(reason != "commentsDisabled" || behavior == retryOnCommentsDisabled)
|
||||||
{
|
{
|
||||||
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath);
|
return reason == "playlistNotFound" && behavior == returnErrorIfPlaylistNotFound ? data : getJson(threadId, url, true, directoryPath);
|
||||||
|
Loading…
Reference in New Issue
Block a user