vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Change database dump format + add import & export scripts

Browse source 
The new format allows for downloading and importing only a part of the
database - useful when only the metadata is required - and doesn't
include the wasteful preformatted HTML cache.

This also ensures that the new import.sql script is actually usable and
in sync with the actual database. The old schema.sql was neither.

(And this simplifies my backup scripts)
This commit is contained in:
Yorhel 2025-10-08 09:40:50 +02:00
parent 23b2686672
commit 902048e282
8 changed files with 273 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

3
util/cron.sh
View file

@ -16,3 +16,6 @@ $PSQL -f update_indices.sql
echo "============ Updating HTML cache"
test -f .config && source ./.config
./cache-html.pl --batch=5 --delay=0.5 --maxbatches=100
echo "============ Updating database dumps"
./export.sh

34
util/export.sh Executable file
View file

@ -0,0 +1,34 @@
#!/bin/sh
set -e
mkdir -p ../dl
# Only run once a week
[ -e ../dl/current ] && [ -z $(find ../dl/current -daystart -mtime +7) ] && exit
# Only keep the last dump
rm -rf $(printf '%s\n' ../dl/????-??-?? | sort | head -n -1)
# Create a new dump
OUT=../dl/.work
export OUT
rm -fr $OUT
mkdir -p $OUT
cp ../import.sql $OUT/import.sql
psql -wqU manned <<EOF
\copy systems to program 'zstd - -qo $OUT/systems.tsv.zst'
\copy contents (id, hash, content) to program 'zstd - -qo $OUT/contents.tsv.zst'
\copy mans to program 'zstd - -qo $OUT/mans.tsv.zst'
\copy locales to program 'zstd - -qo $OUT/locales.tsv.zst'
\copy encodings to program 'zstd - -qo $OUT/encodings.tsv.zst'
\copy packages to program 'zstd - -qo $OUT/packages.tsv.zst'
\copy package_versions to program 'zstd - -qo $OUT/package_versions.tsv.zst'
\copy files to program 'zstd - -qo $OUT/files.tsv.zst'
EOF
DATE=$(date +%F)
mv -T $OUT ../dl/$DATE
echo $DATE >../dl/current

2
util/update_indices.sql
View file

@ -6,7 +6,7 @@ CREATE TABLE stats_cache_new AS
SELECT (SELECT count(*) FROM contents) AS hashes,
(SELECT count(distinct name) FROM mans) AS mans, *
FROM (SELECT count(*), count(distinct pkgver) FROM files) x(files, packages);
DROP TABLE stats_cache;
DROP TABLE IF EXISTS stats_cache;
ALTER TABLE stats_cache_new RENAME TO stats_cache;
COMMIT;