generate-backups.sh

#!/usr/bin/env bash

set -e

LOGS=$1
DATABASES=$2
SCALE=$3
PARTITIONS=$4

DATADIR=/mnt/pgdata/data
DEBUG=$LOGS/debug.log
BACKUPS=/mnt/pgdata/backups

PERCENT="1 15 35"

RUNS=1

PATH=/var/lib/postgresql/builds/pg-master/bin:$PATH

# backup index
backup=0
manifest="-"

killall postgres || true
sleep 5


echo `date` initializing cluster

# init and start the cluster
rm -Rf $DATADIR $BACKUPS $LOGS/pg.log $DEBUG

pg_ctl -D $DATADIR -o "-k" init >> $DEBUG 2>&1
cp postgresql.conf $DATADIR

mkdir $BACKUPS

# maybe enable/disable checksums?

# start the cluster
pg_ctl -D $DATADIR -l $LOGS/pg.log start >> $DEBUG 2>&1

# initialize the databases
echo `date` initializing pgbench databases
for b in $(seq 1 $DATABASES); do
        createdb db-$b
        pgbench -i -s $SCALE --partitions=$PARTITIONS db-$b >> $LOGS/debug.log 2>&1
        echo `date` database db-$b initialized
done


# create initial full basebackup
pg_basebackup -c fast -D $BACKUPS/full >> $DEBUG 2>&1
manifest=$BACKUPS/full/backup_manifest

# now run pgbench on all the databases, with more and more changes
for pct in $PERCENT; do

        for run in $(seq 1 $RUNS); do

                # scale is a multiple of 100k rows in pgbench_accounts, but only ~1640 pages
                # and the backups work at page level, so assume each update hits one random page
                # (ignore collisions)
                ROWS=$((1640 * SCALE * pct / 100))

                # calculate number of clients and transactions per client
                c=$((90/DATABASES))
                t=$((ROWS/DATABASES/c))

                echo `date` "running pgbench - $pct pct run $run ($c clients $t transactions)"

                for b in $(seq 1 $DATABASES); do
                        pgbench -t $t -c $c db-$b >> $LOGS/debug.log 2>&1 &
                done

                # wait for the pgbench runs to complete
                wait

                echo `date` "performing incremental backup"

                echo `date` pg_basebackup -c fast --incremental=$manifest -D $BACKUPS/increment-$backup

                # create another incremental backup
                backup=$((backup+1))
                pg_basebackup -c fast --incremental=$manifest -D $BACKUPS/increment-$backup >> $DEBUG 2>&1
                manifest=$BACKUPS/increment-$backup/backup_manifest

                echo `date` "incremental backup completed"

        done

done

echo `date` stopping cluster
pg_ctl -D $DATADIR stop >> $LOGS/debug.log 2>&1

du -s $BACKUPS/*