#!/bin/sh # Usage: # # read mx_token # mx_token=$mx_token sudo -E synapse-maint set -e -u case "${1:-}" in srun) exec systemd-run --collect --uid=matrix-synapse \ --unit=synapse-maintainance -E mx_token \ --nice=4 -pCPUSchedulingPolicy=batch -pIOSchedulingClass=idle $0 ;; log) shift if test $# -eq 0 then set -- -b --since=-10days fi exec journalctl -u synapse-maintainance "$@" ;; esac PATH=/home/joerg/bin:$PATH db=matrix conn_str="host=/run/postgresql user=$(id -u -n) dbname=$db application_name=state_compress" : ${synapse_admin_url:=http://localhost:8008/_synapse/admin/v1} export RUST_BACKTRACE=short LC_ALL=C.UTF-8 if which time >/dev/null then export TIME='%E elapsed %M rss' time=time else time= fi psql() { command psql -X -b "$@" "dbname=$db application_name=synapse-maint" } psql_t() { ${time:-command} psql -X -b "$@" "dbname=$db application_name=synapse-maint" } if test -n "$mx_token" then echo 'Removing orphaned rooms' echo '=======================' echo for room in $( curl -s --header "Authorization: Bearer $mx_token" "$synapse_admin_url/rooms?limit=800" \ |jq -r '.rooms[] |select(.joined_local_members == 0) | .room_id' ) do echo -n "Purging $room: " $time curl -s -X DELETE -H "Authorization: Bearer $mx_token" \ -H 'Accept: application/json' -H "Content-Type: application/json" \ -d '{"purge": true, "message": "Sorry - kicking you out to clean up the database"}' \ "$synapse_admin_url/rooms/$room" done fi # see _purge_room_txn in synapse/storage/databases/main/purge_events.py (v1.98.0) echo 'Removing data referencing unknown rooms' echo '=======================================' echo for tbl in event_auth event_edges event_json events device_lists_changes_in_room \ state_groups state_groups_state state_events do printf '%s: ' "$tbl" psql_t -c "DELETE FROM $tbl WHERE room_id NOT IN (SELECT room_id FROM rooms);" done # TODO this takes too much time # echo 'Removing data referencing unknown events' # echo '========================================' # echo # for tbl in event_json state_groups state_groups_state # do # printf '%s: ' "$tbl" # psql_t -c "DELETE FROM $tbl WHERE event_id NOT IN (SELECT event_id FROM events);" # done # cargo install --git='https://github.com/matrix-org/rust-synapse-compress-state' synapse_auto_compressor if which synapse_compress_state >/dev/null then echo 'Compressing state' echo '=================' echo cd "$(mktemp -d)" echo "Writing SQL files to $PWD" # SELECT set_config('autovacuum', 'off'', false); SELECT pg_reload_conf(); ? echo 'Disabling autovacuum on state_groups_state' psql -q -c 'alter table state_groups_state set (autovacuum_enabled = false);' for room in $( psql -t -A \ -c 'SELECT room_id FROM state_groups GROUP BY room_id ORDER BY count(*)' ) do echo '--------------------' echo sql_file=state-compress-$room.sql $time synapse_compress_state -t -p "$conn_str" -o $sql_file -r $room -m 1000 if test -s $sql_file then psql_t -q -c '\set ON_ERROR_STOP on' -f $sql_file else rm $sql_file fi done echo 'Enabling autovacuum on state_groups_state' psql -q -c 'alter table state_groups_state set (autovacuum_enabled = true);' echo 'Running VACUUM and ANALYZE for state_groups_state ...' psql_t -q -c 'VACUUM FULL ANALYZE state_groups_state' echo "All SQL scripts are in $PWD" fi psql_t -q -c "REINDEX DATABASE CONCURRENTLY $db"