run-with-deadlock-detection.sh

#!/usr/bin/bash
#
# Runs crash-test-with-retries-for-schema.sh with randomized parameters in a loop, and ensures
# each run completes within 15 minutes (to prevent infinite lockups).
#
# The randomized parameters are
#
# number of tables - 10 to 110
# refresh interval - 10 to 110 tables
# sleep length     - 1 to 11 seconds
#
# Each run stores data in a separate directory, derived from timestamp.
#

RUNS=$1
TIMEOUT=900

#for v in 12 14 16 master; do
for v in master; do

	PUB_VERSION=$v
	SUB_VERSION=$v

	for r in `seq 1 $RUNS`; do

		t=$((10 + RANDOM % 100))	# number of tables
		m=$((10 + RANDOM % 100))	# publication refresh interval
		s=$((1 + RANDOM % 10))		# sleep between steps (seconds)

		OUTDIR=`date +%Y%m%d-%H%M%S`

		if [ -d "$OUTDIR" ]; then
			echo "directory $OUTDIR already exists"
			exit
		fi

		mkdir $OUTDIR

		# make sure the test is terminated after 1h (or change 
		sleep $TIMEOUT && kill `pgrep crash-test-with-retries-for-schema.sh` &
		pid=$!

		# run the test with random parameters
		./crash-test-with-retries-for-schema.sh $OUTDIR $t $m  $s 5 30 > $OUTDIR/run.log 2>&1
		r=$?

		kill $pid

		# make sure it's dead
		killall -9 -q postgres

		if [ "$r" == "0" ]; then
			echo $OUTDIR $t $m $s : OK
		elif [ "$r" == "1" ]; then
			echo $OUTDIR $t $m $s : ERROR
		else
			echo $OUTDIR $t $m $s : DEADLOCK
		fi

	done

done