#2: Add compression to channels/ folder

Can use following Python script to compress existing uncompressed
`channels/` folder.

```py

import os, shutil

path = 'channels/'

os.chdir(path)

d = next(os.walk('.'))[1]
for channelIndex, channelId in enumerate(d):
    print(f'{channelIndex} / {len(d)}: {channelId}')
    shutil.make_archive(channelId, 'zip', channelId)
    shutil.rmtree(channelId)
```
This commit is contained in:
Benjamin Loison 2023-01-04 03:06:33 +01:00
parent a4a282642d
commit 512485b1b8

View File

@ -17,8 +17,10 @@ void createDirectory(string path),
print(ostringstream* toPrint),
treatComment(unsigned short threadId, json comment, string channelId),
treatChannelOrVideo(unsigned short threadId, bool isChannel, string id, string channelToTreat),
treatChannels(unsigned short threadId);
string getHttps(string url);
treatChannels(unsigned short threadId),
deleteDirectory(string path);
string getHttps(string url),
exec(string cmd);
size_t writeCallback(void* contents, size_t size, size_t nmemb, void* userp);
bool doesFileExist(string filePath),
writeFile(unsigned short threadId, string filePath, string option, string toWrite);
@ -100,6 +102,10 @@ void treatChannels(unsigned short threadId)
treatChannelOrVideo(threadId, true, channelToTreat, channelToTreat);
// As I haven't found any well-known library that compress easily a directory, I have chosen to rely on `zip` cli.
exec("cd " + channelToTreatDirectory + " && zip -r ../" + channelToTreat + ".zip *");
deleteDirectory(channelToTreatDirectory);
PRINT(threadId, commentsCount << " comments were found for this channel.")
commentsCount = 0;
requestsPerChannel = 0;
@ -229,6 +235,22 @@ void treatComment(unsigned short threadId, json comment, string channelId)
commentsCount++;
}
string exec(string cmd)
{
array<char, 128> buffer;
string result;
unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
if (!pipe)
{
throw runtime_error("popen() failed!");
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
{
result += buffer.data();
}
return result;
}
bool writeFile(unsigned short threadId, string filePath, string option, string toWrite)
{
FILE* file = fopen(filePath.c_str(), option.c_str());
@ -256,6 +278,11 @@ void createDirectory(string path)
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
}
void deleteDirectory(string path)
{
filesystem::remove_all(path);
}
string getDate()
{
auto t = time(nullptr);