restore-backups.sh

#!/usr/bin/env bash

set -e

LOGS=$1
DATABASES=$2
SCALE=$3
PARTITIONS=$4

RESTOREDIR=/mnt/pgdata/restored
DEBUG=debug.log
BACKUPS=/mnt/pgdata/backups
PATH_OLD=$PATH
RUNS=1

# number of incremental-backups
NUM_BACKUPS=$(ls $BACKUPS | grep increment | wc -l)

for p in 0-baseline; do
#for p in 7-copy-chunks 1-alignment 2-prefetching 6-clone-checksums 3-basic-cloning 5-advanced-reconstruct 4-basic-reconstruct; do

        mkdir $LOGS/$p
        rm -Rf $RESTOREDIR/restore-*
        PATH=/var/lib/postgresql/builds/pg-master-$p/bin:$PATH_OLD

        backups="$BACKUPS/full"

        for b in $(seq 1 $NUM_BACKUPS); do

                backups="$backups $BACKUPS/increment-$b"

                for method in copy copy-file-range; do

                        for manifest in on off; do

                                OPTIONS=""

                                if [ "$manifest" == "off" ]; then
                                        OPTIONS="$OPTIONS --no-manifest"
                                fi

                                if [ "$method" == "copy-file-range" ]; then

                                        # first two options don't have the copy-file-range option
                                        if [ "$p" == "0-baseline" ] || [ "$p" == "1-alignment" ] || [ "$p" == "2-prefetching" ]; then
                                                continue
                                        fi

                                        OPTIONS="$OPTIONS --copy-file-range"
                                fi

                                for r in $(seq 1 $RUNS); do

                                        echo `date` "build $p combining backups method $method manifest $manifest ($backups)"

                                        rm -Rf $RESTOREDIR/restore-*

                                        echo `date` "fstrim"
                                        sudo fstrim /mnt/pgdata || true

                                        sync
                                        used=$(df /mnt/pgdata | grep pgdata | awk '{print $3}')

                                        echo `date` $(which pg_combinebackup)

                                        echo `date` pg_combinebackup -o $RESTOREDIR/restore-$b-$r $OPTIONS $backups

                                        s=$(date +"%s.%6N")
                                        pg_combinebackup -o $RESTOREDIR/restore-$b-$r $OPTIONS $backups > $LOGS/$p/debug-$b-$r.log 2>&1
                                        e=$(date +"%s.%6N")
                                        d=$(echo "$e - $s" | bc)

                                        echo `date` backups combined

                                        verify=""
                                        if [ "$manifest" == "on" ]; then
                                                echo `date` verifying combined backup
                                                pg_verifybackup -P -e -m $RESTOREDIR/restore-$b-$r/backup_manifest $RESTOREDIR/restore-$b-$r > $LOGS/$p/verify-$b-$r.log 2>&1
                                                verify=$?
                                                echo `date` backup verified
                                        fi

                                        echo `date` finishing recovery

                                        # restart the cluster, so finish recovery before checksums check
                                        pg_ctl -D $RESTOREDIR/restore-$b-$r -l $LOGS/$p/restore-$b-$r.log start > /dev/null 2>&1
                                        pg_ctl -D $RESTOREDIR/restore-$b-$r -l $LOGS/$p/restore-$b-$r.log stop > /dev/null 2>&1

                                        sync
                                        size=$(df /mnt/pgdata | grep pgdata | awk '{print $3}')

                                        zpool get allocated,bcloneratio,bclonesaved,bcloneused

                                        echo `date` recovery finished, verifying checksums

                                        # verify checksums
                                        pg_checksums --check $RESTOREDIR/restore-$b-$r > $LOGS/$p/checksums-$b-$r.log 2>&1
                                        checksums=$?

                                        echo `date` "checksums verified ($checksums)"

                                        #echo `date` "calculading checksum of restored database"

                                        # restart the cluster, so finish recovery before checksums check
                                        #pg_ctl -D $RESTOREDIR/restore-$b-$r -l $LOGS/$p/restore-$b-$r.log start > /dev/null 2>&1
                                        #chsum=$(pg_dumpall | md5sum | awk '{print $1}')
                                        #pg_ctl -D $RESTOREDIR/restore-$b-$r -l $LOGS/$p/restore-$b-$r.log stop > /dev/null 2>&1

                                        #echo `date` "checksum of restored database: $chsum"

                                        echo `date` "combinebackup done: " $((size-used)) "KB, $d seconds, checksums $checksums verify $verify"

                                        echo $DATABASES $SCALE $PARTITIONS $p $manifest $method $b $r $d $((size-used)) $checksums >> $LOGS/results.csv 2>&1

                                done

                        done

                done

        done

done