0006_warm_taptests_v21.patch

diff --git b/src/test/modules/warm/t/001_recovery.pl a/src/test/modules/warm/t/001_recovery.pl
new file mode 100644
index 0000000..2a76830
--- /dev/null
+++ a/src/test/modules/warm/t/001_recovery.pl
@@ -0,0 +1,50 @@
+# Single-node test: run workload, crash, recover and run sanity check
+
+use strict;
+use warnings;
+
+use TestLib;
+use Test::More tests => 2;
+use PostgresNode;
+
+my $node = get_new_node();
+$node->init;
+$node->start;
+
+# Create a table, do some WARM updates and then restart
+$node->safe_psql('postgres',
+	'create table accounts (aid int unique, branch int, balance bigint) with (fillfactor=98)');
+$node->safe_psql('postgres',
+	'create table history (aid int, delta int)');
+$node->safe_psql('postgres',
+	'insert into accounts select generate_series(1,10000), (random()*1000)::int % 10, 0');
+$node->safe_psql('postgres',
+	'create index accounts_bal_indx on accounts(balance)');
+
+for( $a = 1; $a <= 1000; $a = $a + 1 ) {
+	my $aid1 = int(rand(10000)) + 1;
+	my $aid2 = int(rand(10000)) + 1;
+	my $balance = int(rand(99999));
+	$node->safe_psql('postgres',
+		"begin;
+		 update accounts set balance = balance + $balance where aid = $aid1;
+		 update accounts set balance = balance - $balance where aid = $aid2;
+		 insert into history values ($aid1, $balance);
+		 insert into history values ($aid2, 0 - $balance);
+		 end;");
+}
+
+# Verify that we read the same TS after crash recovery
+$node->stop('immediate');
+$node->start;
+
+my $recovered_balance = $node->safe_psql('postgres', 'select sum(balance) from accounts');
+my $total_delta = $node->safe_psql('postgres', 'select sum(delta) from history');
+
+# since delta is credited to one account and debited from the other, we expect
+# the sum(balance) to stay zero.
+is($recovered_balance, 0, 'balanace matches after recovery');
+
+# A positive and a negative value is inserted in the history table. Hence the
+# sum(delta) should remain zero.
+is($total_delta, 0, 'sum(delta) matches after recovery');
diff --git b/src/test/modules/warm/t/002_warm_stress.pl a/src/test/modules/warm/t/002_warm_stress.pl
new file mode 100644
index 0000000..a1a2371
--- /dev/null
+++ a/src/test/modules/warm/t/002_warm_stress.pl
@@ -0,0 +1,289 @@
+# Run varity of tests to check consistency of index access.
+#
+# These tests are primarily designed to test if WARM updates cause any
+# inconsistency in the indexes. We use a pgbench-like setup with an "accounts"
+# table and a "branches" table. But instead of a single "aid" column the
+# pgbench_warm_accounts table has four additional columns. These columns have
+# initial value as "aid * 10", "aid * 20", "aid * 30" and "aid * 40". And
+# unlike the aid column, values in these columns do not remain static. The
+# values are changed in a narrow change around the original value, such that
+# they still remain distinct, even after updates. We also build indexes on
+# these additional columns.
+#
+# This allows us to force WARM updates to the table, while accessing individual
+# rows using these auxillary columns. If things are solid, we must not miss any
+# row irrespective of which column we use to fetch the row. Also, the sum of
+# balances in two tables should match at the end.
+#
+# We drop and recreate indexes concurrently and also run VACUUM and run
+# consistency checks to ensure nothing breaks. The tests also aborts
+# transactions, acquires share/update locks etc to check any negative effects
+# of those things.
+
+use strict;
+use warnings;
+
+use TestLib;
+use Test::More tests => 10;
+use PostgresNode;
+
+# Different kinds of queries, some committing, some aborting. Also include FOR
+# SHARE, FOR UPDATE which may have implications on the visibility bits etc.
+my @query_set1 = (
+
+	"begin;
+	update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid;
+	select abalance from pgbench_warm_accounts where aid = :aid;
+	update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid;
+	end;",
+
+	"begin;
+	update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid;
+	select abalance from pgbench_warm_accounts where aid = :aid;
+	update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid;
+	rollback;",
+
+	"begin;
+	select abalance from pgbench_warm_accounts where aid = :aid for update;
+	update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid;
+	select bbalance from pgbench_warm_branches where bid = :bid for update;
+	update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid;
+	commit;",
+
+	"begin;
+	select abalance from pgbench_warm_accounts where aid = :aid for update;
+	update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid;
+	select bbalance from pgbench_warm_branches where bid = :bid for update;
+	update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid;
+	rollback;",
+
+	"begin;
+	select abalance from pgbench_warm_accounts where aid = :aid for share;
+	update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid;
+	select bbalance from pgbench_warm_branches where bid = :bid for update;
+	update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid;
+	commit;",
+
+	"begin;
+	select abalance from pgbench_warm_accounts where aid = :aid for update;
+	update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid;
+	select bbalance from pgbench_warm_branches where bid = :bid for update;
+	update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid;
+	rollback;"
+);
+
+# The following queries use user-defined functions to update rows in
+# pgbench_warm_accounts table by using auxillary columns. This allows us to
+# test if the updates are working fine in various scenarios.
+my @query_set2 = (
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid1(:chg1, :aid, :bid, :delta);
+	commit;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid2(:chg2, :aid, :bid, :delta);
+	commit;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid3(:chg3, :aid, :bid, :delta);
+	commit;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid4(:chg4, :aid, :bid, :delta);
+	commit;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid1(:chg1, :aid, :bid, :delta);
+	rollback;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid2(:chg2, :aid, :bid, :delta);
+	rollback;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid3(:chg3, :aid, :bid, :delta);
+	rollback;",
+
+	"begin;
+	set enable_seqscan to false;
+	select pgbench_warm_update_using_aid4(:chg4, :aid, :bid, :delta);
+	rollback;"
+);
+
+# Specify concurrent DDLs that you may want to execute with the tests.
+my @ddl_queries = (
+	"drop index pgb_a_aid1;
+	 create index pgb_a_aid1 on pgbench_warm_accounts(aid1);",
+	"drop index pgb_a_aid2;
+	 create index pgb_a_aid2 on pgbench_warm_accounts(aid2);",
+	"drop index pgb_a_aid3;
+	 create index pgb_a_aid3 on pgbench_warm_accounts(aid3) using hash (aid3);",
+	"drop index pgb_a_aid4;
+	 create index pgb_a_aid4 on pgbench_warm_accounts(aid4);",
+	"drop index pgb_a_aid1;
+	 create index concurrently pgb_a_aid1 on pgbench_warm_accounts(aid1);",
+	"drop index pgb_a_aid2;
+	 create index concurrently pgb_a_aid2 on pgbench_warm_accounts(aid2);",
+	"drop index pgb_a_aid3;
+	 create index concurrently pgb_a_aid3 on pgbench_warm_accounts(aid3) using hash(aid3);",
+	"drop index pgb_a_aid4;
+	 create index concurrently pgb_a_aid4 on pgbench_warm_accounts(aid4);",
+	"vacuum pgbench_warm_accounts",
+	"vacuum pgbench_warm_branches",
+	"vacuum full pgbench_warm_accounts",
+	"vacuum full pgbench_warm_branches"
+);
+
+# Consistency check queries.
+my @check_queries = (
+	"set enable_seqscan to false; select pgbench_warm_check_consistency();",
+	"set enable_seqscan to false; select pgbench_warm_check_row(:aid);"
+);
+
+my $node = get_new_node();
+$node->init;
+$node->start;
+
+# prepare the test for execution
+$node->run_log([ 'psql', '-X', $node->connstr(), '-f', 't/warm_stress_init.sql']);
+
+my $res = $node->safe_psql('postgres', "select proname from pg_proc where proname = 'pgbench_warm_update_using_aid1'");
+is($res, 'pgbench_warm_update_using_aid1', 'dummy test passed');
+
+$res = $node->safe_psql('postgres', "select count(*) from pgbench_warm_accounts");
+is($res, 10000, 'Fine match');
+
+# Start as many connections as we need
+sub create_connections {
+	my $count = shift;
+	my @handles;
+	my ($stdin, $stdout, $stderr) = ('','','');
+	for (my $proc = 0; $proc < $count; $proc = $proc + 1) {
+		my $handle = IPC::Run::start( 
+			[
+				'psql', '-v', '-f -', $node->connstr(),
+			],
+			\$stdin, \$stdout, \$stderr);
+		push @handles, [$handle,\$stdin,\$stdout,\$stderr];
+	}
+	return \@handles;
+}
+
+sub check_connections {
+	my @handles = @_;
+	my $failures = 0;
+	print @handles;
+	foreach my $elem (@handles) {
+		my ($handle, $stdin, $stdout, $stderr) = @$elem;
+		# Wait for all queries to complete and psql sessions to exit, checking
+		# exit codes. We don't need to do the fancy interpretation safe_psql
+		# does.
+		$handle->finish;
+		if (!is($handle->full_result(0), 0, "psql exited normally"))
+		{
+			$failures ++;
+			diag "psql exit code: " . ($handle->result(0)) . " or signal: " . ($handle->full_result(0) & 127);
+			diag "Stdout:\n---\n$$stdout\n---\nStderr:\n----\n$$stderr\n---";
+		}
+	}
+	return $failures;
+}
+
+my $set1_handles = create_connections(3);
+my $set2_handles = create_connections(3);
+my $aux_handles = create_connections(1);
+
+# Run a few thousand transactions, using various kinds of queries
+my $scale = 1;
+for (my $txn = 0; $txn < 10000; $txn = $txn + 1) {
+	# Run a randomly chosen query from set1
+	my $aid = int(rand($scale*10000)) + 1;
+	my $bid = int(rand(100)) + 1;
+	my $delta = int(rand(1000)) - 500;
+
+	my $connindx = rand(@$set1_handles);
+	my $elem = @$set1_handles[$connindx];
+	my ($handle, $stdin, $stdout, $stderr) = @$elem;
+
+	my $queryindx = rand(@query_set1);
+	my $query = $query_set1[$queryindx];
+
+	$query =~ s/\:aid/$aid/g;
+	$query =~ s/\:bid/$bid/g;
+	$query =~ s/\:delta/$delta/g;
+
+	$$stdin .= $query . "\n";
+	pump $handle while length $$stdin;
+
+	# Run a randomly chosen query from set1
+	my $chg1 = int(rand(4)) - 2;
+	my $chg2 = int(rand(6)) - 3;
+	my $chg3 = int(rand(8)) - 4;
+	my $chg4 = int(rand(10)) - 5;
+
+	$connindx = rand(@$set2_handles);
+	$elem = @$set2_handles[$connindx];
+	($handle, $stdin, $stdout, $stderr) = @$elem;
+
+	$queryindx = rand(@query_set2);
+	$query = $query_set2[$queryindx];
+
+	$query =~ s/\:aid/$aid/g;
+	$query =~ s/\:bid/$bid/g;
+	$query =~ s/\:delta/$delta/g;
+	$query =~ s/\:chg1/$chg1/g;
+	$query =~ s/\:chg2/$chg2/g;
+	$query =~ s/\:chg3/$chg3/g;
+	$query =~ s/\:chg4/$chg4/g;
+
+	$$stdin .= $query . "\n";
+	pump $handle while length $$stdin;
+
+	# Some randomly picked numbers to run DDLs and consistency checks
+	my $random = int(rand(100));
+
+	# Consistenct checks every 5 transactions
+	if ($random % 5 == 0)
+	{
+		$connindx = rand(@$aux_handles);
+		$elem = @$aux_handles[$connindx];
+		($handle, $stdin, $stdout, $stderr) = @$elem;
+
+		$queryindx = rand(@check_queries);
+		$query = $check_queries[$queryindx];
+
+		$$stdin .= $query . "\n";
+		pump $handle while length $$stdin;
+	}
+
+	# 1% DDLs
+	if ($random == 17)
+	{
+		$connindx = rand(@$aux_handles);
+		$elem = @$aux_handles[$connindx];
+		($handle, $stdin, $stdout, $stderr) = @$elem;
+
+		$queryindx = rand(@ddl_queries);
+		$query = $ddl_queries[$queryindx];
+
+		$$stdin .= $query . "\n";
+		pump $handle while length $$stdin;
+	}
+}
+
+check_connections(@$set1_handles);
+check_connections(@$set2_handles);
+check_connections(@$aux_handles);
+
+# Run final consistency checks
+my $res1 = $node->safe_psql('postgres', "select sum(abalance) from pgbench_warm_accounts");
+my $res2 = $node->safe_psql('postgres', "select sum(bbalance) from pgbench_warm_branches");
+is($res1, $res2, 'Fine match');
diff --git b/src/test/modules/warm/t/warm_stress_init.sql a/src/test/modules/warm/t/warm_stress_init.sql
new file mode 100644
index 0000000..4697480
--- /dev/null
+++ a/src/test/modules/warm/t/warm_stress_init.sql
@@ -0,0 +1,209 @@
+
+drop table if exists pgbench_warm_branches;
+drop table if exists pgbench_warm_accounts;
+
+create table pgbench_warm_branches (
+	bid bigint,
+	bbalance bigint);
+
+create table pgbench_warm_accounts (
+	aid bigint,
+	bid bigint,
+	abalance bigint,
+	aid1 bigint ,
+	aid2 bigint ,
+	aid3 bigint ,
+	aid4 bigint ,
+	aid5 text default md5(random()::text),
+	aid6 text default md5(random()::text),
+	aid7 text default md5(random()::text),
+	aid8 text default md5(random()::text),
+	aid9 text default md5(random()::text),
+	aid10 text default md5(random()::text),
+	gistcol	polygon default null
+);
+
+-- update using aid1. aid1 should stay within the range (aid * 10 - 2 <= aid1 <= aid * 10 + 2) 
+create or replace function pgbench_warm_update_using_aid1(chg integer, v_aid bigint, v_bid bigint, delta bigint)
+returns void as $$
+declare
+	qry varchar;
+	lower varchar;
+	upper varchar;
+	range integer;
+	aid_updated bigint;
+begin
+	range := 2;
+	update pgbench_warm_accounts p set aid1 = aid1 +  chg,  abalance = abalance +
+delta  where aid1 >= v_aid * 10 - range - chg and aid1 <= v_aid * 10 + range - chg
+returning p.aid into aid_updated;
+	if aid_updated is not null then
+		update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid;
+	else
+		select aid into aid_updated from pgbench_warm_accounts p where aid1 >=
+v_aid * 10 - range and aid1 <= v_aid * 10 + range;
+		if aid_updated is null then
+			raise exception 'pgbench_warm_accounts row not found';
+		end if;
+	end if;
+end
+$$ language plpgsql;
+
+-- update using aid2. aid2 should stay within the range (aid * 20 - 4 <= aid2 <= aid * 20 + 4) 
+create or replace function pgbench_warm_update_using_aid2(chg integer, v_aid bigint, v_bid bigint, delta bigint)
+returns void as $$
+declare
+	qry varchar;
+	lower varchar;
+	upper varchar;
+	range integer;
+	aid_updated bigint;
+begin
+	range := 4;
+	update pgbench_warm_accounts p set aid2 = aid2 +  chg,  abalance = abalance +
+delta  where aid2 >= v_aid * 20 - range - chg and aid2 <= v_aid * 20 + range - chg
+returning p.aid into aid_updated;
+	if aid_updated is not null then
+		update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid;
+	else
+		select aid into aid_updated from pgbench_warm_accounts p where aid2 >= v_aid * 20 - range and aid2 <= v_aid * 20 + range;
+		if aid_updated is null then
+			raise exception 'pgbench_warm_accounts row not found';
+		end if;
+	end if;
+end
+$$ language plpgsql;
+
+-- update using aid3. aid3 should stay within the range (aid * 30 - 6 <= aid3 <= aid * 30 + 6) 
+create or replace function pgbench_warm_update_using_aid3(chg integer, v_aid bigint, v_bid bigint, delta bigint)
+returns void as $$
+declare
+	qry varchar;
+	lower varchar;
+	upper varchar;
+	range integer;
+	aid_updated bigint;
+begin
+	range := 6;
+	update pgbench_warm_accounts p set aid3 = aid3 +  chg,  abalance = abalance +
+delta  where aid3 >= v_aid * 30 - range - chg and aid3 <= v_aid * 30 + range - chg
+returning p.aid into aid_updated;
+	if aid_updated is not null then
+		update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid;
+	else
+		select aid into aid_updated from pgbench_warm_accounts p where aid3 >= v_aid * 30 - range and aid3 <= v_aid * 30 + range;
+		if aid_updated is null then
+			raise exception 'pgbench_warm_accounts row not found';
+		end if;
+	end if;
+end
+$$ language plpgsql;
+
+-- update using aid4. aid4 should stay within the range (aid * 40 - 8 <= aid4 <= aid * 40 + 8) 
+create or replace function pgbench_warm_update_using_aid4(chg integer, v_aid bigint, v_bid bigint, delta bigint)
+returns void as $$
+declare
+	qry varchar;
+	lower varchar;
+	upper varchar;
+	range integer;
+	aid_updated bigint;
+begin
+	range := 8;
+	update pgbench_warm_accounts p set aid4 = aid4 +  chg,  abalance = abalance +
+delta  where aid4 >= v_aid * 40 - range - chg and aid4 <= v_aid * 40 + range - chg
+returning p.aid into aid_updated;
+	if aid_updated is not null then
+		update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid;
+	else
+		select aid into aid_updated from pgbench_warm_accounts p where aid4 >= v_aid * 40 - range and aid4 <= v_aid * 40 + range;
+		if aid_updated is null then
+			raise exception 'pgbench_warm_accounts row not found';
+		end if;
+	end if;
+end
+$$ language plpgsql;
+
+-- ensure that exactly one row exists within a given range. use different
+-- indexes to fetch the row
+create or replace function pgbench_warm_check_row(v_aid bigint)
+returns void as $$
+declare
+	range integer;
+	factor integer;
+	ret_aid1 bigint;
+	ret_aid2 bigint;
+	ret_aid3 bigint;
+	ret_aid4 bigint;
+begin
+	range := 2;
+	factor := 10;
+	select aid into ret_aid1 from pgbench_warm_accounts p where aid1 >= v_aid *
+		factor - range and aid1 <= v_aid * factor + range;
+
+	range := 4;
+	factor := 20;
+	select aid into ret_aid2 from pgbench_warm_accounts p where aid2 >= v_aid *
+		factor - range and aid2 <= v_aid * factor + range;
+
+	range := 6;
+	factor := 30;
+	select aid into ret_aid3 from pgbench_warm_accounts p where aid3 >= v_aid *
+		factor - range and aid3 <= v_aid * factor + range;
+
+	range := 8;
+	factor := 40;
+	select aid into ret_aid4 from pgbench_warm_accounts p where aid4 >= v_aid *
+		factor - range and aid4 <= v_aid * factor + range;
+
+	if ret_aid1 is null or ret_aid1 != v_aid then
+		raise exception 'pgbench_warm_accounts row (%) not found via aid1', v_aid;
+	end if;
+
+	if ret_aid2 is null or ret_aid2 != v_aid then
+		raise exception 'pgbench_warm_accounts row (%) not found via aid2', v_aid;
+	end if;
+
+	if ret_aid3 is null or ret_aid3 != v_aid then
+		raise exception 'pgbench_warm_accounts row (%) not found via aid3', v_aid;
+	end if;
+
+	if ret_aid4 is null or ret_aid4 != v_aid then
+		raise exception 'pgbench_warm_accounts row (%) not found via aid4', v_aid;
+	end if;
+end
+$$ language plpgsql;
+
+create or replace function pgbench_warm_check_consistency()
+returns void as $$
+declare
+	sum_abalance bigint;
+	sum_bbalance bigint;
+begin
+	select sum(abalance) into sum_abalance from pgbench_warm_accounts;
+	select sum(bbalance) into sum_bbalance from pgbench_warm_branches;
+	if sum_abalance != sum_bbalance then
+		raise exception 'found inconsitency in sum (%, %)', sum_abalance, sum_bbalance;
+	end if;
+end
+$$ language plpgsql;
+
+\set end 10000
+insert into pgbench_warm_branches select generate_series(1, 100), 0 ;
+insert into pgbench_warm_accounts select generate_series(1, :end),
+				(random() * 100)::int, 0,
+				generate_series(1, :end) * 10,
+				generate_series(1, :end) * 20,
+				generate_series(1, :end) * 30,
+				generate_series(1, :end) * 40;
+
+create unique index pgb_a_aid on pgbench_warm_accounts(aid);
+create index pgb_a_aid1 on pgbench_warm_accounts(aid1);
+create index pgb_a_aid2 on pgbench_warm_accounts(aid2);
+create index pgb_a_aid3 on pgbench_warm_accounts(aid3) using hash(aid3);
+create index pgb_a_aid4 on pgbench_warm_accounts(aid4);
+
+create unique index pgb_b_bid on pgbench_warm_branches(bid);
+create index pgb_b_bbalance on pgbench_warm_branches(bbalance);
+
+vacuum analyze;