Change database dump format + add import & export scripts
The new format allows for downloading and importing only a part of the database - useful when only the metadata is required - and doesn't include the wasteful preformatted HTML cache. This also ensures that the new import.sql script is actually usable and in sync with the actual database. The old schema.sql was neither. (And this simplifies my backup scripts)
This commit is contained in:
parent
23b2686672
commit
902048e282
8 changed files with 273 additions and 141 deletions
34
util/export.sh
Executable file
34
util/export.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
mkdir -p ../dl
|
||||
|
||||
# Only run once a week
|
||||
[ -e ../dl/current ] && [ -z $(find ../dl/current -daystart -mtime +7) ] && exit
|
||||
|
||||
# Only keep the last dump
|
||||
rm -rf $(printf '%s\n' ../dl/????-??-?? | sort | head -n -1)
|
||||
|
||||
# Create a new dump
|
||||
OUT=../dl/.work
|
||||
export OUT
|
||||
rm -fr $OUT
|
||||
mkdir -p $OUT
|
||||
|
||||
cp ../import.sql $OUT/import.sql
|
||||
|
||||
psql -wqU manned <<EOF
|
||||
\copy systems to program 'zstd - -qo $OUT/systems.tsv.zst'
|
||||
\copy contents (id, hash, content) to program 'zstd - -qo $OUT/contents.tsv.zst'
|
||||
\copy mans to program 'zstd - -qo $OUT/mans.tsv.zst'
|
||||
\copy locales to program 'zstd - -qo $OUT/locales.tsv.zst'
|
||||
\copy encodings to program 'zstd - -qo $OUT/encodings.tsv.zst'
|
||||
\copy packages to program 'zstd - -qo $OUT/packages.tsv.zst'
|
||||
\copy package_versions to program 'zstd - -qo $OUT/package_versions.tsv.zst'
|
||||
\copy files to program 'zstd - -qo $OUT/files.tsv.zst'
|
||||
EOF
|
||||
|
||||
DATE=$(date +%F)
|
||||
mv -T $OUT ../dl/$DATE
|
||||
echo $DATE >../dl/current
|
||||
Loading…
Add table
Add a link
Reference in a new issue