#2: Add compression to channels/ folder

Can use following Python script to compress existing uncompressed
`channels/` folder.

```py

import os, shutil

path = 'channels/'

os.chdir(path)

d = next(os.walk('.'))[1]
for channelIndex, channelId in enumerate(d):
    print(f'{channelIndex} / {len(d)}: {channelId}')
    shutil.make_archive(channelId, 'zip', channelId)
    shutil.rmtree(channelId)
```
This commit is contained in:
Benjamin Loison 2023-01-04 03:06:33 +01:00
parent f201ae7a91
commit 9d5c9fde2a
Signed by: Benjamin_Loison
SSH Key Fingerprint: SHA256:BtnEgYTlHdOg1u+RmYcDE0mnfz1rhv5dSbQ2gyxW8B8

View File

@ -17,8 +17,10 @@ void createDirectory(string path),
print(ostringstream* toPrint), print(ostringstream* toPrint),
treatComment(unsigned short threadId, json comment, string channelId), treatComment(unsigned short threadId, json comment, string channelId),
treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat), treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat),
treatChannels(unsigned short threadId); treatChannels(unsigned short threadId),
string getHttps(string url); deleteDirectory(string path);
string getHttps(string url),
exec(string cmd);
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp); size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath), bool doesFileExist(string filePath),
writeFile(unsigned short threadId, string filePath, string option, string toWrite); writeFile(unsigned short threadId, string filePath, string option, string toWrite);
@ -100,6 +102,10 @@ void treatChannels(unsigned short threadId)
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat); treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
exec("cd " + channelToTreatDirectory + " && zip -r ../" + channelToTreat + ".zip *");
deleteDirectory(channelToTreatDirectory);
PRINT(threadId, commentsCount << " comments were found for this channel.") PRINT(threadId, commentsCount << " comments were found for this channel.")
commentsCount = 0; commentsCount = 0;
requestsPerChannel = 0; requestsPerChannel = 0;
@ -229,6 +235,22 @@ void treatComment(unsigned short threadId, json comment, string channelId)
commentsCount++; commentsCount++;
} }
string exec(string cmd)
{
array<char, 128> buffer;
string result;
unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
if (!pipe)
{
throw runtime_error("popen() failed!");
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
{
result += buffer.data();
}
return result;
}
bool writeFile(unsigned short threadId, string filePath, string option, string toWrite) bool writeFile(unsigned short threadId, string filePath, string option, string toWrite)
{ {
FILE* file = fopen(filePath.c_str(), option.c_str()); FILE* file = fopen(filePath.c_str(), option.c_str());
@ -256,6 +278,11 @@ void createDirectory(string path)
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
} }
void deleteDirectory(string path)
{
filesystem::remove_all(path);
}
string getDate() string getDate()
{ {
auto t = time(nullptr); auto t = time(nullptr);