Skip to content

Commit e41635e

Browse files
linkmauvelinkmauve
committed
Switch exports to Zstd, from bzip2
The main benefit of this format for our users is that it decompresses much faster than bzip2, even at high compression levels. At level 19 it compresses even better than bzip2 for our files, hopefully the compression time is still acceptable, if not we can reduce it as to not overwork the server, at the price of some slightly bigger files. On my i7-8700K, unarchiving sentences.tar.bz2 takes 15.5s, compared to 994ms for sentences.csv.zst compressed at level 19. The file is 183 MiB compared to 197 MiB with bzip2. We could go down to 167 MiB with level 22 (which decompresses in 941ms), but compression time starts to get much higher, not sure this is worth it. The only downside I see to this change is that user automation will have to be changed, so perhaps announce it somehow before deploying it.
1 parent 3206771 commit e41635e

5 files changed

Lines changed: 41 additions & 41 deletions

File tree

‎ansible/roles/setup_database/tasks/import_csv.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@
1919
url: "{{ download_url }}{{ item }}"
2020
dest: "/tmp/{{ item }}"
2121
with_items:
22-
- sentences.tar.bz2
23-
- links.tar.bz2
22+
- sentences.csv.zst
23+
- links.csv.zst
2424
- tag_metadata.csv
25-
- tags_detailed.tar.bz2
25+
- tags_detailed.csv.zst
2626
when: import_csv == 'download'
2727

2828
- name: Unpacking csv's
@@ -34,9 +34,9 @@
3434
owner: mysql
3535
group: mysql
3636
with_items:
37-
- sentences.tar.bz2
38-
- links.tar.bz2
39-
- tags_detailed.tar.bz2
37+
- sentences.csv.zst
38+
- links.csv.zst
39+
- tags_detailed.csv.zst
4040
when: import_csv == 'download'
4141

4242
- name: Import sentences in the tatoeba database

‎docs/cron/export.sh‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,26 @@ mv /var/tmp/*csv "$DL_DIR"
1919

2020
echo "Starting tarring at $(date -Iseconds)"
2121
cd "$DL_DIR"
22-
tar -cjf sentences_base.tar.bz2 sentences_base.csv
23-
tar -cjf sentences_detailed.tar.bz2 sentences_detailed.csv
24-
tar -cjf links.tar.bz2 links.csv
25-
tar -cjf sentences.tar.bz2 sentences.csv
26-
tar -cjf contributions.tar.bz2 contributions.csv
22+
zstd -19 sentences_base.csv
23+
zstd -19 sentences_detailed.csv
24+
zstd -19 links.csv
25+
zstd -19 sentences.csv
26+
zstd -19 contributions.csv
2727
rm contributions.csv
28-
tar -cjf comments.tar.bz2 sentence_comments.csv
28+
zstd -19 sentence_comments.csv
2929
rm sentence_comments.csv
30-
tar -cjf wall.tar.bz2 wall_posts.csv
30+
zstd -19 wall_posts.csv
3131
rm wall_posts.csv
32-
tar -cjf tags.tar.bz2 tags.csv
33-
tar -cjf user_lists.tar.bz2 user_lists.csv
34-
tar -cjf sentences_in_lists.tar.bz2 sentences_in_lists.csv
35-
tar -cjf jpn_indices.tar.bz2 jpn_indices.csv
36-
tar -cjf sentences_with_audio.tar.bz2 sentences_with_audio.csv
37-
tar -cjf user_languages.tar.bz2 user_languages.csv
38-
tar -cjf tags_detailed.tar.bz2 tags_detailed.csv
39-
tar -cjf sentences_CC0.tar.bz2 sentences_CC0.csv
40-
tar -cjf transcriptions.tar.bz2 transcriptions.csv
41-
tar -cjf sentences_base.tar.bz2 sentences_base.csv
32+
zstd -19 tags.csv
33+
zstd -19 user_lists.csv
34+
zstd -19 sentences_in_lists.csv
35+
zstd -19 jpn_indices.csv
36+
zstd -19 sentences_with_audio.csv
37+
zstd -19 user_languages.csv
38+
zstd -19 tags_detailed.csv
39+
zstd -19 sentences_CC0.csv
40+
zstd -19 transcriptions.csv
41+
zstd -19 sentences_base.csv
4242

4343
echo "Starting language splitting for sentences at $(date -Iseconds)"
4444
# Create per-language files for the different sentences files
@@ -173,7 +173,7 @@ mysql --skip-column-names --batch tatoeba -e \
173173
}'
174174

175175
echo "Starting cleanup at $(date -Iseconds)"
176-
find $TEMP_DIR -path '*tsv' -exec bzip2 -qf '{}' +
176+
find $TEMP_DIR -path '*tsv' -exec zstd -19 -qf '{}' +
177177
rm -rf $DL_DIR/per_language
178178
rm transcriptions.csv
179179
mv -f $TEMP_DIR $DL_DIR

‎src/Template/Pages/downloads.ctp‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
273273
<dl>
274274
<dt><?= $filename ?></dt>
275275
<dd>
276-
<a href="<?= $download_url ?>sentences_base.tar.bz2">sentences_base.tar.bz2</a>
276+
<a href="<?= $download_url ?>sentences_base.csv.zst">sentences_base.csv.zst</a>
277277
</dd>
278278
<dt><?= $description ?></dt>
279279
<dd>
@@ -323,7 +323,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
323323
<dl>
324324
<dt><?= $filename ?></dt>
325325
<dd>
326-
<a href="<?= $download_url ?>links.tar.bz2">links.tar.bz2</a>
326+
<a href="<?= $download_url ?>links.csv.zst">links.csv.zst</a>
327327
</dd>
328328
<dt><?= $description ?></dt>
329329
<dd>
@@ -350,7 +350,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
350350
<dl>
351351
<dt><?= $filename ?></dt>
352352
<dd>
353-
<a href="<?= $download_url ?>tags.tar.bz2">tags.tar.bz2</a>
353+
<a href="<?= $download_url ?>tags.csv.zst">tags.csv.zst</a>
354354
</dd>
355355
<dt><?= $description ?></dt>
356356
<dd>
@@ -376,7 +376,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
376376
<dl>
377377
<dt><?= $filename ?></dt>
378378
<dd>
379-
<a href="<?= $download_url ?>user_lists.tar.bz2">user_lists.tar.bz2</a>
379+
<a href="<?= $download_url ?>user_lists.csv.zst">user_lists.csv.zst</a>
380380
</dd>
381381
<dt><?= $description ?></dt>
382382
<dd>
@@ -400,8 +400,8 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
400400
<dl>
401401
<dt><?= $filename ?></dt>
402402
<dd>
403-
<a href="<?= $download_url ?>sentences_in_lists.tar.bz2">
404-
sentences_in_lists.tar.bz2
403+
<a href="<?= $download_url ?>sentences_in_lists.csv.zst">
404+
sentences_in_lists.csv.zst
405405
</a>
406406
</dd>
407407
<dt><?= $description ?></dt>
@@ -429,7 +429,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
429429
<dl>
430430
<dt><?= $filename ?></dt>
431431
<dd>
432-
<a href="<?= $download_url ?>jpn_indices.tar.bz2">jpn_indices.tar.bz2</a>
432+
<a href="<?= $download_url ?>jpn_indices.csv.zst">jpn_indices.csv.zst</a>
433433
</dd>
434434
<dt><?= $description ?></dt>
435435
<dd>
@@ -460,8 +460,8 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
460460
<dl>
461461
<dt><?= $filename ?></dt>
462462
<dd>
463-
<a href="<?= $download_url ?>sentences_with_audio.tar.bz2">
464-
sentences_with_audio.tar.bz2
463+
<a href="<?= $download_url ?>sentences_with_audio.csv.zst">
464+
sentences_with_audio.csv.zst
465465
</a>
466466
</dd>
467467
<dt><?= $description ?></dt>
@@ -501,7 +501,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
501501
<dl>
502502
<dt><?= $filename ?></dt>
503503
<dd>
504-
<a href="<?= $download_url ?>user_languages.tar.bz2">user_languages.tar.bz2</a>
504+
<a href="<?= $download_url ?>user_languages.csv.zst">user_languages.csv.zst</a>
505505
</dd>
506506
<dt><?= $description ?></dt>
507507
<dd>

‎src/View/Helper/DownloadsHelper.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ private function availableFiles($basename) {
4848
);
4949

5050
$dir = new Folder($perLanguageDir);
51-
$paths = $dir->findRecursive(".*$basename\.tsv\.bz2$");
51+
$paths = $dir->findRecursive(".*$basename\.tsv\.zst$");
5252
$map = [];
5353
foreach ($paths as $path) {
5454
$path = substr($path, strlen($perLanguageDir) + 1);
@@ -74,7 +74,7 @@ private function availableFiles($basename) {
7474
public function createOptions($basename) {
7575
$urlForAll = Folder::addPathElement(
7676
Configure::read('Downloads.url'),
77-
"$basename.tar.bz2"
77+
"$basename.csv.zst"
7878
);
7979
$options[0] = [
8080
'language' => __('All languages'),

‎tests/TestCase/View/Helper/DownloadsHelperTest.php‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ class DownloadsHelperTest extends TestCase {
1414

1515
private static function createTempTree() {
1616
$languages = ['eng', 'fra', 'jpn', 'unknown'];
17-
$files = ['sentences.tsv.bz2', 'sentences_detailed.tsv.bz2', 'sentences_CC0.tsv.bz2'];
17+
$files = ['sentences.tsv.zst', 'sentences_detailed.tsv.zst', 'sentences_CC0.tsv.zst'];
1818
foreach ($languages as $lang) {
1919
$path = Folder::addPathElement(TMP, ['exports', 'per_language', $lang]);
2020
$subdir = new Folder($path, true);
@@ -54,7 +54,7 @@ public function testCreateOptions_InvalidBasename() {
5454

5555
$this->assertEquals(1, count($options));
5656
$this->assertEquals(
57-
Folder::addPathElement(Configure::read('Downloads.url'), "foobar.tar.bz2"),
57+
Folder::addPathElement(Configure::read('Downloads.url'), "foobar.csv.zst"),
5858
$options[0]['url']
5959
);
6060
}
@@ -77,14 +77,14 @@ public function testCreateOptions_ValidBasename($basename) {
7777
$this->assertEquals(
7878
Folder::addPathElement(
7979
Configure::read('Downloads.url'),
80-
"$basename.tar.bz2"
80+
"$basename.csv.zst"
8181
),
8282
$options[0]['url']
8383
);
8484
$this->assertEquals(
8585
Folder::addPathElement(
8686
Configure::read('Downloads.url'),
87-
['per_language', 'eng', "eng_$basename.tsv.bz2"]
87+
['per_language', 'eng', "eng_$basename.tsv.zst"]
8888
),
8989
$options[1]['url']
9090
);
@@ -103,7 +103,7 @@ public function testCreateOptions_NoPerLanguageFilesAvailable($basename) {
103103
$this->assertEquals(
104104
Folder::addPathElement(
105105
Configure::read('Downloads.url'),
106-
"$basename.tar.bz2"
106+
"$basename.csv.zst"
107107
),
108108
$options[0]['url']
109109
);

0 commit comments

Comments
 (0)