0003-Poc-buffer-encryption.patch

From fba051add0f0ec73770db415bba5a1c486d41c27 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Mon, 11 Jan 2021 19:37:12 +0900
Subject: [PATCH 3/3] Poc: buffer encryption.

---
 contrib/bloom/blinsert.c                |   1 +
 src/backend/access/hash/hashpage.c      |   1 +
 src/backend/access/heap/rewriteheap.c   |   1 +
 src/backend/access/nbtree/nbtree.c      |   1 +
 src/backend/access/nbtree/nbtsort.c     |   1 +
 src/backend/access/spgist/spginsert.c   |   3 +
 src/backend/access/transam/xloginsert.c |  13 +++
 src/backend/bootstrap/bootstrap.c       |   3 +
 src/backend/catalog/storage.c           |   2 +-
 src/backend/crypto/Makefile             |   1 +
 src/backend/crypto/bufenc.c             | 138 ++++++++++++++++++++++++
 src/backend/postmaster/postmaster.c     |   2 +
 src/backend/storage/buffer/bufmgr.c     |  25 ++++-
 src/backend/storage/page/bufpage.c      |  49 ++++++++-
 src/backend/tcop/postgres.c             |   2 +
 src/common/cipher_openssl.c             |  26 +++++
 src/include/access/xloginsert.h         |   1 +
 src/include/common/cipher.h             |   3 +-
 src/include/crypto/bufenc.h             |  27 +++++
 src/include/crypto/kmgr.h               |   2 +
 src/include/storage/bufpage.h           |  11 +-
 21 files changed, 306 insertions(+), 7 deletions(-)
 create mode 100644 src/backend/crypto/bufenc.c
 create mode 100644 src/include/crypto/bufenc.h

diff --git a/contrib/bloom/blinsert.c b/contrib/bloom/blinsert.c
index 32b5d62e1f..d474af753c 100644
--- a/contrib/bloom/blinsert.c
+++ b/contrib/bloom/blinsert.c
@@ -177,6 +177,7 @@ blbuildempty(Relation index)
 	 * XLOG_DBASE_CREATE or XLOG_TBLSPC_CREATE record.  Therefore, we need
 	 * this even when wal_level=minimal.
 	 */
+	PageEncryptInplace(metapage, INIT_FORKNUM, BLOOM_METAPAGE_BLKNO);
 	PageSetChecksumInplace(metapage, BLOOM_METAPAGE_BLKNO);
 	smgrwrite(index->rd_smgr, INIT_FORKNUM, BLOOM_METAPAGE_BLKNO,
 			  (char *) metapage, true);
diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c
index 49a9867787..7ee505c6ba 100644
--- a/src/backend/access/hash/hashpage.c
+++ b/src/backend/access/hash/hashpage.c
@@ -1025,6 +1025,7 @@ _hash_alloc_buckets(Relation rel, BlockNumber firstblock, uint32 nblocks)
 					true);
 
 	RelationOpenSmgr(rel);
+	PageEncryptInplace(page, MAIN_FORKNUM, lastblock);
 	PageSetChecksumInplace(page, lastblock);
 	smgrextend(rel->rd_smgr, MAIN_FORKNUM, lastblock, zerobuf.data, false);
 
diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c
index 29ffe40670..d752331706 100644
--- a/src/backend/access/heap/rewriteheap.c
+++ b/src/backend/access/heap/rewriteheap.c
@@ -325,6 +325,7 @@ end_heap_rewrite(RewriteState state)
 						true);
 		RelationOpenSmgr(state->rs_new_rel);
 
+		PageEncryptInplace(state->rs_buffer, MAIN_FORKNUM, state->rs_blockno);
 		PageSetChecksumInplace(state->rs_buffer, state->rs_blockno);
 
 		smgrextend(state->rs_new_rel->rd_smgr, MAIN_FORKNUM, state->rs_blockno,
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index ba79a7f3e9..b2969dd07e 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -175,6 +175,7 @@ btbuildempty(Relation index)
 	 * XLOG_DBASE_CREATE or XLOG_TBLSPC_CREATE record.  Therefore, we need
 	 * this even when wal_level=minimal.
 	 */
+	PageEncryptInplace(metapage, INIT_FORKNUM, BTREE_METAPAGE);
 	PageSetChecksumInplace(metapage, BTREE_METAPAGE);
 	smgrwrite(index->rd_smgr, INIT_FORKNUM, BTREE_METAPAGE,
 			  (char *) metapage, true);
diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c
index c904fc8ef7..04da2d865b 100644
--- a/src/backend/access/nbtree/nbtsort.c
+++ b/src/backend/access/nbtree/nbtsort.c
@@ -665,6 +665,7 @@ _bt_blwritepage(BTWriteState *wstate, Page page, BlockNumber blkno)
 				   true);
 	}
 
+	PageEncryptInplace(page, MAIN_FORKNUM, blkno);
 	PageSetChecksumInplace(page, blkno);
 
 	/*
diff --git a/src/backend/access/spgist/spginsert.c b/src/backend/access/spgist/spginsert.c
index 2e1d8a33d1..84187583d2 100644
--- a/src/backend/access/spgist/spginsert.c
+++ b/src/backend/access/spgist/spginsert.c
@@ -168,6 +168,7 @@ spgbuildempty(Relation index)
 	 * of their existing content when the corresponding create records are
 	 * replayed.
 	 */
+	PageEncryptInplace(page, INIT_FORKNUM, SPGIST_METAPAGE_BLKNO);
 	PageSetChecksumInplace(page, SPGIST_METAPAGE_BLKNO);
 	smgrwrite(index->rd_smgr, INIT_FORKNUM, SPGIST_METAPAGE_BLKNO,
 			  (char *) page, true);
@@ -177,6 +178,7 @@ spgbuildempty(Relation index)
 	/* Likewise for the root page. */
 	SpGistInitPage(page, SPGIST_LEAF);
 
+	PageEncryptInplace(page, INIT_FORKNUM, SPGIST_ROOT_BLKNO);
 	PageSetChecksumInplace(page, SPGIST_ROOT_BLKNO);
 	smgrwrite(index->rd_smgr, INIT_FORKNUM, SPGIST_ROOT_BLKNO,
 			  (char *) page, true);
@@ -186,6 +188,7 @@ spgbuildempty(Relation index)
 	/* Likewise for the null-tuples root page. */
 	SpGistInitPage(page, SPGIST_LEAF | SPGIST_NULLS);
 
+	PageEncryptInplace(page, INIT_FORKNUM, SPGIST_NULL_BLKNO);
 	PageSetChecksumInplace(page, SPGIST_NULL_BLKNO);
 	smgrwrite(index->rd_smgr, INIT_FORKNUM, SPGIST_NULL_BLKNO,
 			  (char *) page, true);
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 7052dc245e..d7738c8834 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -1190,6 +1190,19 @@ log_newpage_range(Relation rel, ForkNumber forkNum,
 	}
 }
 
+/* Write a no-op dummy xlog */
+XLogRecPtr
+log_noop(void)
+{
+	bool dummy;
+
+	XLogBeginInsert();
+
+	/* register dummy data as the record have to contain data */
+	XLogRegisterData((char *) &dummy, sizeof(bool));
+	return XLogInsert(RM_XLOG_ID, XLOG_NOOP);
+}
+
 /*
  * Allocate working buffers needed for WAL record construction.
  */
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index e2231f1430..9eba3735a5 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_collation.h"
 #include "catalog/pg_type.h"
 #include "common/link-canary.h"
+#include "crypto/bufenc.h"
 #include "crypto/kmgr.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -528,6 +529,8 @@ BootstrapModeMain(void)
 
 	InitPostgres(NULL, InvalidOid, NULL, InvalidOid, NULL, false);
 
+	InitializeBufferEncryption();
+
 	/* Initialize stuff for bootstrap-file processing */
 	for (i = 0; i < MAXATTR; i++)
 	{
diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index cba7a9ada0..c876c536a5 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -443,7 +443,7 @@ RelationCopyStorage(SMgrRelation src, SMgrRelation dst,
 
 		smgrread(src, forkNum, blkno, buf.data);
 
-		if (!PageIsVerifiedExtended(page, blkno,
+		if (!PageIsVerifiedExtended(page, forkNum, blkno,
 									PIV_LOG_WARNING | PIV_REPORT_STAT))
 			ereport(ERROR,
 					(errcode(ERRCODE_DATA_CORRUPTED),
diff --git a/src/backend/crypto/Makefile b/src/backend/crypto/Makefile
index c27362029d..8985a66875 100644
--- a/src/backend/crypto/Makefile
+++ b/src/backend/crypto/Makefile
@@ -13,6 +13,7 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 OBJS = \
+	bufenc.o \
 	kmgr.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/crypto/bufenc.c b/src/backend/crypto/bufenc.c
new file mode 100644
index 0000000000..4401303fe9
--- /dev/null
+++ b/src/backend/crypto/bufenc.c
@@ -0,0 +1,138 @@
+/*-------------------------------------------------------------------------
+ *
+ * bufenc.c
+ *
+ * Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ *	  src/backend/crypto/bufenc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "miscadmin.h"
+#include "lib/stringinfo.h"
+
+#include "storage/bufpage.h"
+#include "storage/fd.h"
+#include "crypto/kmgr.h"
+#include "crypto/bufenc.h"
+
+/*-------------------------------------------------------------------------
+ * We use both page LSN and page number to create a nonce for each page. Page
+ * LSN is 8 byte, page number is 4 byte, and the maximum required counter for
+ * AES-CTR is 2048, which fits in 3 byte. Since the length of IV is 16 byte
+ * it's fine. Using the LSN and page number as part of the nonce has
+ * three benefits:
+ *
+ * 1. We don't need to decrypt/re-encrypt during CREATE DATABASE since the page
+ * contents are the same in both places, and once one database changes its pages,
+ * it gets a new LSN, and hence a new nonce.
+ * 2. For each change of an 8k page, we get a new nonce, so we are not encrypting
+ * different data with the same nonce/IV.
+ * 3. We avoid requiring pg_upgrade to preserve database oids, tablespace oids,
+ * relfilenodes.
+ *
+ * Different two pages could have the same page LSN when heap update expires the
+ * old tuple and add the new tuple to the another page.  By using the page number
+ * as a part of nonce we can get an unique nonce even in this case. We assume that
+ * we don't use the same LSN in different relations.
+ *-------------------------------------------------------------------------
+ */
+#define BUFENC_IV_SIZE		16
+
+static unsigned char buf_encryption_iv[BUFENC_IV_SIZE];
+
+PgCipherCtx *BufencCtx = NULL;
+PgCipherCtx *BufdecCtx = NULL;
+
+static void set_buffer_encryption_iv(Page page, BlockNumber blkno);
+
+void
+InitializeBufferEncryption(void)
+{
+	const CryptoKey *key;
+
+	if (!DataEncryptionEnabled())
+		return;
+
+	key = KmgrGetKey(KMGR_KEY_ID_REL);
+
+	BufencCtx = pg_cipher_ctx_create(PG_CIPHER_AES_CTR,
+									 (unsigned char *) key->key,
+									 (file_encryption_keylen / 8), true);
+	if (!BufencCtx)
+		elog(ERROR, "could not intialize encryption context");
+
+	BufdecCtx = pg_cipher_ctx_create(PG_CIPHER_AES_CTR,
+									 (unsigned char *) key->key,
+									 (file_encryption_keylen / 8), false);
+	if (!BufdecCtx)
+		elog(ERROR, "could not intialize decryption context");
+}
+
+/* Encrypt the given page with the relation key */
+void
+EncryptPage(Page page, BlockNumber blkno)
+{
+	unsigned char *ptr = (unsigned char *) page + PageEncryptOffset;
+	int enclen;
+
+	Assert(BufencCtx != NULL);
+
+	set_buffer_encryption_iv(page, blkno);
+	if (unlikely(!pg_cipher_encrypt(BufencCtx,
+									(const unsigned char *) ptr,	/* input  */
+									SizeOfPageEncryption,
+									ptr,	/* length */
+									&enclen,	/* resulting length */
+									buf_encryption_iv,	/* iv */
+									BUFENC_IV_SIZE,
+									NULL, 0)))
+		elog(ERROR, "could not encrypt page %u", blkno);
+	Assert(enclen == SizeOfPageEncryption);
+}
+
+/* Decrypt the given page with the relation key */
+void
+DecryptPage(Page page, BlockNumber blkno)
+{
+	unsigned char *ptr = (unsigned char *) page + PageEncryptOffset;
+	int enclen;
+
+	Assert(BufdecCtx != NULL);
+
+	set_buffer_encryption_iv(page, blkno);
+	if (unlikely(!pg_cipher_decrypt(BufdecCtx,
+									(const unsigned char *) ptr,	/* input  */
+									SizeOfPageEncryption,
+									ptr,	/* output */
+									&enclen,	/* resulting length */
+									buf_encryption_iv,	/* iv */
+									BUFENC_IV_SIZE,
+									NULL, 0)))
+		elog(ERROR, "could not decrypt page %u", blkno);
+	Assert(enclen == SizeOfPageEncryption);
+}
+
+/* Construct iv for the given page */
+static void
+set_buffer_encryption_iv(Page page, BlockNumber blkno)
+{
+	unsigned char *p = buf_encryption_iv;
+
+	MemSet(buf_encryption_iv, 0, BUFENC_IV_SIZE);
+
+	/* page lsn (8 byte) */
+	memcpy(p, &((PageHeader) page)->pd_lsn, sizeof(PageXLogRecPtr));
+	p += sizeof(PageXLogRecPtr);
+
+	/* block number (4 byte) */
+	memcpy(p, &blkno, sizeof(BlockNumber));
+	p += sizeof(BlockNumber);
+
+	/* remainig 4 bytes are the space for the counter, already set to 0 */
+}
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index e2f8e07c88..c0424f3f36 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -100,6 +100,7 @@
 #include "common/file_perm.h"
 #include "common/ip.h"
 #include "common/string.h"
+#include "crypto/bufenc.h"
 #include "crypto/kmgr.h"
 #include "lib/ilist.h"
 #include "libpq/auth.h"
@@ -1333,6 +1334,7 @@ PostmasterMain(int argc, char *argv[])
 	RemovePgTempFiles();
 
 	InitializeKmgr();
+	InitializeBufferEncryption();
 
 	if (terminal_fd != -1)
 		close(terminal_fd);
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 71b5852224..c86b6ea068 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -37,6 +37,7 @@
 #include "access/xlog.h"
 #include "catalog/catalog.h"
 #include "catalog/storage.h"
+#include "crypto/bufenc.h"
 #include "executor/instrument.h"
 #include "lib/binaryheap.h"
 #include "miscadmin.h"
@@ -918,7 +919,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 			}
 
 			/* check for garbage data */
-			if (!PageIsVerifiedExtended((Page) bufBlock, blockNum,
+			if (!PageIsVerifiedExtended((Page) bufBlock, forkNum, blockNum,
 										PIV_LOG_WARNING | PIV_REPORT_STAT))
 			{
 				if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages)
@@ -2792,12 +2793,15 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln)
 	 */
 	bufBlock = BufHdrGetBlock(buf);
 
+	bufToWrite = PageEncryptCopy((Page) bufBlock, buf->tag.forkNum,
+								 buf->tag.blockNum);
+
 	/*
 	 * Update page checksum if desired.  Since we have only shared lock on the
 	 * buffer, other processes might be updating hint bits in it, so we must
 	 * copy the page to private storage if we do checksumming.
 	 */
-	bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
+	bufToWrite = PageSetChecksumCopy((Page) bufToWrite, buf->tag.blockNum);
 
 	if (track_io_timing)
 		INSTR_TIME_SET_CURRENT(io_start);
@@ -3280,6 +3284,8 @@ FlushRelationBuffers(Relation rel)
 				errcallback.previous = error_context_stack;
 				error_context_stack = &errcallback;
 
+				PageEncryptInplace(localpage, bufHdr->tag.forkNum,
+								   bufHdr->tag.blockNum);
 				PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
 
 				smgrwrite(rel->rd_smgr,
@@ -3677,6 +3683,21 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std)
 		{
 			dirtied = true;		/* Means "will be dirtied by this action" */
 
+			/*
+			 * We will dirty the page but the page lsn is not changed if we
+			 * doesn't write a backup block. We don't want to encrypt the
+			 * different bits stream with the same combination of nonce and key
+			 * since in buffer encryption the page lsn is a part of nonce.
+			 * Therefore we WAL-log no-op record just to move page lsn forward if
+			 * we doesn't write a backup block, even when this is not the first
+			 * modification in this checkpoint round.
+			 */
+			if (XLogRecPtrIsInvalid(lsn) && DataEncryptionEnabled())
+			{
+				lsn = log_noop();
+				Assert(!XLogRecPtrIsInvalid(lsn));
+			}
+
 			/*
 			 * Set the page LSN if we wrote a backup block. We aren't supposed
 			 * to set this when only holding a share lock but as long as we
diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c
index 9ac556b4ae..3a480e7b82 100644
--- a/src/backend/storage/page/bufpage.c
+++ b/src/backend/storage/page/bufpage.c
@@ -17,6 +17,7 @@
 #include "access/htup_details.h"
 #include "access/itup.h"
 #include "access/xlog.h"
+#include "crypto/bufenc.h"
 #include "pgstat.h"
 #include "storage/checksum.h"
 #include "utils/memdebug.h"
@@ -85,7 +86,8 @@ PageInit(Page page, Size pageSize, Size specialSize)
  * to pgstat.
  */
 bool
-PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags)
+PageIsVerifiedExtended(Page page, ForkNumber forknum, BlockNumber blkno,
+					   int flags)
 {
 	PageHeader	p = (PageHeader) page;
 	size_t	   *pagebytes;
@@ -108,6 +110,9 @@ PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags)
 				checksum_failure = true;
 		}
 
+		if (DataEncryptionEnabled())
+			PageDecryptInplace(page, forknum, blkno);
+
 		/*
 		 * The following checks don't prove the header is correct, only that
 		 * it looks sane enough to allow into the buffer pool. Later usage of
@@ -1427,3 +1432,45 @@ PageSetChecksumInplace(Page page, BlockNumber blkno)
 
 	((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno);
 }
+
+char *
+PageEncryptCopy(Page page, ForkNumber forknum, BlockNumber blkno)
+{
+	static char *pageCopy = NULL;
+
+	/* If we don't need a checksum, just return the passed-in data */
+	if (PageIsNew(page) || !PageNeedsToBeEncrypted(forknum))
+		return (char *) page;
+
+	/*
+	 * We allocate the copy space once and use it over on each subsequent
+	 * call.  The point of palloc'ing here, rather than having a static char
+	 * array, is first to ensure adequate alignment for the checksumming code
+	 * and second to avoid wasting space in processes that never call this.
+	 */
+	if (pageCopy == NULL)
+		pageCopy = MemoryContextAlloc(TopMemoryContext, BLCKSZ);
+
+	memcpy(pageCopy, (char *) page, BLCKSZ);
+	EncryptPage(pageCopy, blkno);
+	return pageCopy;
+}
+
+void
+PageEncryptInplace(Page page, ForkNumber forknum, BlockNumber blkno)
+{
+	if (PageIsNew(page) || !PageNeedsToBeEncrypted(forknum))
+		return;
+
+	EncryptPage(page, blkno);
+}
+
+
+void
+PageDecryptInplace(Page page, ForkNumber forknum, BlockNumber blkno)
+{
+	if (PageIsNew(page) || PageNeedsToBeEncrypted(forknum))
+		return;
+
+	DecryptPage(page, blkno);
+}
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 0463da6d46..36759c0190 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -43,6 +43,7 @@
 #include "commands/async.h"
 #include "commands/prepare.h"
 #include "crypto/kmgr.h"
+#include "crypto/bufenc.h"
 #include "executor/spi.h"
 #include "jit/jit.h"
 #include "libpq/libpq.h"
@@ -3978,6 +3979,7 @@ PostgresMain(int argc, char *argv[],
 	if (!IsUnderPostmaster)
 	{
 		InitializeKmgr();
+		InitializeBufferEncryption();
 
 		if (terminal_fd != -1)
 			close(terminal_fd);
diff --git a/src/common/cipher_openssl.c b/src/common/cipher_openssl.c
index 28f613cecc..9a6b4ddbc1 100644
--- a/src/common/cipher_openssl.c
+++ b/src/common/cipher_openssl.c
@@ -34,6 +34,7 @@ static ossl_EVP_cipher_func get_evp_aes_gcm(int klen);
 static ossl_EVP_cipher_func get_evp_aes_kw(int klen);
 static EVP_CIPHER_CTX *ossl_cipher_ctx_create(int cipher, unsigned char *key, int klen,
 											  bool enc);
+static ossl_EVP_cipher_func get_evp_aes_ctr(int klen);
 
 /*
  * Return a newly created cipher context.  'cipher' specifies cipher algorithm
@@ -343,6 +344,26 @@ get_evp_aes_kwp(int klen)
 	}
 }
 
+/*
+ * Returns the correct cipher functions for OpenSSL based
+ * on the key length requested.
+ */
+static ossl_EVP_cipher_func
+get_evp_aes_ctr(int klen)
+{
+	switch (klen)
+	{
+		case PG_AES128_KEY_LEN:
+			return EVP_aes_128_ctr;
+		case PG_AES192_KEY_LEN:
+			return EVP_aes_192_ctr;
+		case PG_AES256_KEY_LEN:
+			return EVP_aes_256_ctr;
+		default:
+			return NULL;
+	}
+}
+
 /*
  * Initialize and return an EVP_CIPHER_CTX. Returns NULL if the given
  * cipher algorithm is not supported or on failure.
@@ -389,6 +410,11 @@ ossl_cipher_ctx_create(int cipher, unsigned char *key, int klen, bool enc)
 			if (!func)
 				goto failed;
 			break;
+		case PG_CIPHER_AES_CTR:
+			func = get_evp_aes_ctr(klen);
+			if (!func)
+				goto failed;
+			break;
 		default:
 			goto failed;
 	}
diff --git a/src/include/access/xloginsert.h b/src/include/access/xloginsert.h
index f1d8c39edf..43c4f1f611 100644
--- a/src/include/access/xloginsert.h
+++ b/src/include/access/xloginsert.h
@@ -59,6 +59,7 @@ extern void log_newpages(RelFileNode *rnode, ForkNumber forkNum, int num_pages,
 extern XLogRecPtr log_newpage_buffer(Buffer buffer, bool page_std);
 extern void log_newpage_range(Relation rel, ForkNumber forkNum,
 							  BlockNumber startblk, BlockNumber endblk, bool page_std);
+extern XLogRecPtr log_noop(void);
 extern XLogRecPtr XLogSaveBufferForHint(Buffer buffer, bool buffer_std);
 
 extern void InitXLogInsert(void);
diff --git a/src/include/common/cipher.h b/src/include/common/cipher.h
index 7d446e86b6..9c4a630cd6 100644
--- a/src/include/common/cipher.h
+++ b/src/include/common/cipher.h
@@ -27,7 +27,8 @@
 #define PG_CIPHER_AES_GCM			0
 #define PG_CIPHER_AES_KW			1
 #define PG_CIPHER_AES_KWP			2
-#define PG_MAX_CIPHER_ID			3
+#define PG_CIPHER_AES_CTR			3
+#define PG_MAX_CIPHER_ID			4
 
 /* AES128/192/256 various length definitions */
 #define PG_AES128_KEY_LEN			(128 / 8)
diff --git a/src/include/crypto/bufenc.h b/src/include/crypto/bufenc.h
new file mode 100644
index 0000000000..97302a4f3f
--- /dev/null
+++ b/src/include/crypto/bufenc.h
@@ -0,0 +1,27 @@
+/*-------------------------------------------------------------------------
+ *
+ * bufenc.h
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/crypto/bufenc.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef BUFENC_H
+#define BUFENC_H
+
+#include "storage/bufmgr.h"
+#include "crypto/kmgr.h"
+
+#define DataEncryptionEnabled() (file_encryption_keylen > 0)
+
+/* Cluster encryption encrypts only main fork */
+#define PageNeedsToBeEncrypted(forknum) \
+	(DataEncryptionEnabled() && ((forknum) == MAIN_FORKNUM))
+
+extern void InitializeBufferEncryption(void);
+extern void EncryptPage(Page page, BlockNumber blkno);
+extern void DecryptPage(Page page, BlockNumber blkno);
+
+#endif	/* BUFENC_H */
diff --git a/src/include/crypto/kmgr.h b/src/include/crypto/kmgr.h
index 386ac1cb4a..6a728ebefc 100644
--- a/src/include/crypto/kmgr.h
+++ b/src/include/crypto/kmgr.h
@@ -16,6 +16,8 @@
 #include "storage/relfilenode.h"
 #include "storage/bufpage.h"
 
+#define DataEncryptionEnabled() (file_encryption_keylen > 0)
+
 /* GUC parameters */
 extern int file_encryption_keylen;
 extern char *cluster_key_command;
diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h
index 359b749f7f..fc855cc3b5 100644
--- a/src/include/storage/bufpage.h
+++ b/src/include/storage/bufpage.h
@@ -15,6 +15,7 @@
 #define BUFPAGE_H
 
 #include "access/xlogdefs.h"
+#include "common/relpath.h"
 #include "storage/block.h"
 #include "storage/item.h"
 #include "storage/off.h"
@@ -164,6 +165,8 @@ typedef struct PageHeaderData
 } PageHeaderData;
 
 typedef PageHeaderData *PageHeader;
+#define PageEncryptOffset	offsetof(PageHeaderData, pd_special)
+#define SizeOfPageEncryption (BLCKSZ - PageEncryptOffset)
 
 /*
  * pd_flags contains the following flag bits.  Undefined bits are initialized
@@ -419,7 +422,7 @@ do { \
 						((is_heap) ? PAI_IS_HEAP : 0))
 
 #define PageIsVerified(page, blkno) \
-	PageIsVerifiedExtended(page, blkno, \
+	PageIsVerifiedExtended(page, MAIN_FORKNUM, blkno, \
 						   PIV_LOG_WARNING | PIV_REPORT_STAT)
 
 /*
@@ -433,7 +436,8 @@ StaticAssertDecl(BLCKSZ == ((BLCKSZ / sizeof(size_t)) * sizeof(size_t)),
 				 "BLCKSZ has to be a multiple of sizeof(size_t)");
 
 extern void PageInit(Page page, Size pageSize, Size specialSize);
-extern bool PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags);
+extern bool PageIsVerifiedExtended(Page page, ForkNumber forknum, BlockNumber blkno,
+								   int flags);
 extern OffsetNumber PageAddItemExtended(Page page, Item item, Size size,
 										OffsetNumber offsetNumber, int flags);
 extern Page PageGetTempPage(Page page);
@@ -452,5 +456,8 @@ extern bool PageIndexTupleOverwrite(Page page, OffsetNumber offnum,
 									Item newtup, Size newsize);
 extern char *PageSetChecksumCopy(Page page, BlockNumber blkno);
 extern void PageSetChecksumInplace(Page page, BlockNumber blkno);
+extern char *PageEncryptCopy(Page page, ForkNumber forknum, BlockNumber blkno);
+extern void PageEncryptInplace(Page page, ForkNumber forknum, BlockNumber blkno);
+extern void PageDecryptInplace(Page page, ForkNumber forknum, BlockNumber blkno);
 
 #endif							/* BUFPAGE_H */
-- 
2.27.0