#2: Add compression to channels/
folder
Can use following Python script to compress existing uncompressed `channels/` folder. ```py import os, shutil path = 'channels/' os.chdir(path) d = next(os.walk('.'))[1] for channelIndex, channelId in enumerate(d): print(f'{channelIndex} / {len(d)}: {channelId}') shutil.make_archive(channelId, 'zip', channelId) shutil.rmtree(channelId) ```
This commit is contained in:
parent
a4a282642d
commit
512485b1b8
31
main.cpp
31
main.cpp
@ -17,8 +17,10 @@ void createDirectory(string path),
|
|||||||
print(ostringstream* toPrint),
|
print(ostringstream* toPrint),
|
||||||
treatComment(unsigned short threadId, json comment, string channelId),
|
treatComment(unsigned short threadId, json comment, string channelId),
|
||||||
treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat),
|
treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat),
|
||||||
treatChannels(unsigned short threadId);
|
treatChannels(unsigned short threadId),
|
||||||
string getHttps(string url);
|
deleteDirectory(string path);
|
||||||
|
string getHttps(string url),
|
||||||
|
exec(string cmd);
|
||||||
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
|
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
|
||||||
bool doesFileExist(string filePath),
|
bool doesFileExist(string filePath),
|
||||||
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
|
||||||
@ -100,6 +102,10 @@ void treatChannels(unsigned short threadId)
|
|||||||
|
|
||||||
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
|
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
|
||||||
|
|
||||||
|
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
|
||||||
|
exec("cd " + channelToTreatDirectory + " && zip -r ../" + channelToTreat + ".zip *");
|
||||||
|
deleteDirectory(channelToTreatDirectory);
|
||||||
|
|
||||||
PRINT(threadId, commentsCount << " comments were found for this channel.")
|
PRINT(threadId, commentsCount << " comments were found for this channel.")
|
||||||
commentsCount = 0;
|
commentsCount = 0;
|
||||||
requestsPerChannel = 0;
|
requestsPerChannel = 0;
|
||||||
@ -229,6 +235,22 @@ void treatComment(unsigned short threadId, json comment, string channelId)
|
|||||||
commentsCount++;
|
commentsCount++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string exec(string cmd)
|
||||||
|
{
|
||||||
|
array<char, 128> buffer;
|
||||||
|
string result;
|
||||||
|
unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
|
||||||
|
if (!pipe)
|
||||||
|
{
|
||||||
|
throw runtime_error("popen() failed!");
|
||||||
|
}
|
||||||
|
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
|
||||||
|
{
|
||||||
|
result += buffer.data();
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
bool writeFile(unsigned short threadId, string filePath, string option, string toWrite)
|
bool writeFile(unsigned short threadId, string filePath, string option, string toWrite)
|
||||||
{
|
{
|
||||||
FILE* file = fopen(filePath.c_str(), option.c_str());
|
FILE* file = fopen(filePath.c_str(), option.c_str());
|
||||||
@ -256,6 +278,11 @@ void createDirectory(string path)
|
|||||||
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
|
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void deleteDirectory(string path)
|
||||||
|
{
|
||||||
|
filesystem::remove_all(path);
|
||||||
|
}
|
||||||
|
|
||||||
string getDate()
|
string getDate()
|
||||||
{
|
{
|
||||||
auto t = time(nullptr);
|
auto t = time(nullptr);
|
||||||
|
Loading…
Reference in New Issue
Block a user