0001-Add-encryption-module-supporting-AES-256-by-using-op.patch

From f1c7bbf5af207e433fba1ac5e727a3b5914d52d2 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Fri, 28 Jun 2019 16:26:25 +0900
Subject: [PATCH 1/9] Add encryption module supporting AES-256 by using openssl

This commit introduce new module to encrypt and decrypt data. Curently
this module relis on openssl so is only available when configured with
--with-openssl option.

Currently we support AES-XTS and AES-CTR with 256 bit key
length. AES-XTS can be used for buffer encryption while AES-CTR is
used for WAL encrytion.
---
 src/backend/storage/smgr/Makefile     |   2 +-
 src/backend/storage/smgr/encryption.c | 305 ++++++++++++++++++++++++++++++++++
 src/common/string.c                   |  39 +++++
 src/include/common/string.h           |   1 +
 src/include/storage/encryption.h      |  90 ++++++++++
 5 files changed, 436 insertions(+), 1 deletion(-)
 create mode 100644 src/backend/storage/smgr/encryption.c
 create mode 100644 src/include/storage/encryption.h

diff --git a/src/backend/storage/smgr/Makefile b/src/backend/storage/smgr/Makefile
index e486b7c..765c02a 100644
--- a/src/backend/storage/smgr/Makefile
+++ b/src/backend/storage/smgr/Makefile
@@ -12,6 +12,6 @@ subdir = src/backend/storage/smgr
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = md.o smgr.o
+OBJS = md.o smgr.o encryption.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/storage/smgr/encryption.c b/src/backend/storage/smgr/encryption.c
new file mode 100644
index 0000000..2563369
--- /dev/null
+++ b/src/backend/storage/smgr/encryption.c
@@ -0,0 +1,305 @@
+/*-------------------------------------------------------------------------
+ *
+ * encryption.c
+ *	  This code handles encryption and decryption of data.
+ *
+ * Encryption is done by extension modules loaded by encryption_library GUC.
+ * The extension module must register itself and provide a cryptography
+ * implementation. Key setup is left to the extension module.
+ *
+ *
+ * Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ *	  src/backend/storage/smgr/encryption.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <unistd.h>
+
+#include "common/fe_memutils.h"
+#include "common/sha2.h"
+#include "common/string.h"
+#include "catalog/pg_control.h"
+#include "pgstat.h"
+#include "storage/bufpage.h"
+#include "storage/encryption.h"
+#include "storage/fd.h"
+#include "storage/kmgr.h"
+#include "storage/smgr.h"
+#include "utils/memutils.h"
+#include "miscadmin.h"
+#include "fmgr.h"
+#include "port.h"
+
+#include <openssl/conf.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+
+PGAlignedBlock encrypt_buf;
+
+static bool		encryption_initialized = false;
+static EVP_CIPHER_CTX *ctx_encrypt;
+static EVP_CIPHER_CTX *ctx_decrypt;
+static EVP_CIPHER_CTX *ctx_encrypt_stream;
+static EVP_CIPHER_CTX *ctx_decrypt_stream;
+
+static void setup_encryption_openssl(void);
+static void evp_error(void);
+static void setup_encryption(void) ;
+static void initialize_encryption_context(EVP_CIPHER_CTX **ctx_p, bool stream);
+
+/*
+ * Encryption a buffer block on the given tablespace.
+ */
+void
+EncryptBufferBlock(Oid spcOid, const char *tweak, const char *input,
+				   char *output)
+{
+	char spckey[ENCRYPTION_KEY_SIZE];
+
+	KeyringGetKey(spcOid, spckey);
+
+	/* Always use block cipher for buffer data */
+	encrypt_block(input, output, BLCKSZ, spckey, tweak, false);
+}
+
+/*
+ * Decryption a buffer block on the given tablespace.
+ */
+void
+DecryptBufferBlock(Oid spcOid, const char *tweak, const char *input,
+				   char *output)
+{
+	char spckey[ENCRYPTION_KEY_SIZE];
+
+	KeyringGetKey(spcOid, spckey);
+
+	/* Always use block cipher for buffer data */
+	decrypt_block(input, output, BLCKSZ, spckey, tweak, false);
+}
+
+/*
+ * Encrypts one block of data with a specified tweak value. May only be called
+ * when encryption_enabled is true.
+ *
+ * Input and output buffer may point to the same location.
+ *
+ * "size" must be a (non-zero) multiple of ENCRYPTION_BLOCK.
+ *
+ * "tweak" value must be TWEAK_SIZE bytes long.
+ *
+ * All-zero blocks are not encrypted to correctly handle relation extension,
+ * and also to simplify handling of holes created by seek past EOF and
+ * consequent write (see buffile.c).
+ */
+void
+encrypt_block(const char *input, char *output, Size size,
+			  const char *key, const char *tweak, bool stream)
+{
+	int			out_size;
+	EVP_CIPHER_CTX *ctx;
+
+	Assert((size >= ENCRYPTION_BLOCK_SIZE &&
+			size % ENCRYPTION_BLOCK_SIZE == 0) || stream);
+
+	/* Ensure encryption has setup */
+	if (!encryption_initialized)
+	{
+		setup_encryption();
+		encryption_initialized = true;
+	}
+
+	if (!stream && IsAllZero(input, size))
+	{
+		memset(output, 0, size);
+		return;
+	}
+
+	ctx = !stream ? ctx_encrypt : ctx_encrypt_stream;
+
+	if (EVP_EncryptInit_ex(ctx, NULL, NULL, (unsigned char *) key,
+						   (unsigned char *) tweak) != 1)
+		evp_error();
+
+	if (EVP_EncryptUpdate(ctx, (unsigned char *) output,
+						  &out_size, (unsigned char *) input, size) != 1)
+		evp_error();
+
+	Assert(out_size == size);
+}
+
+/*
+ * Decrypts one block of data with a specified tweak value. May only be called
+ * when encryption_enabled is true.
+ *
+ * Input and output buffer may point to the same location.
+ *
+ * "size" must be a (non-zero) multiple of ENCRYPTION_BLOCK.
+ *
+ * "tweak" value must be ENCRYPTION_TWEAK_SIZE bytes long.
+ *
+ * All-zero blocks are not decrypted to correctly handle relation extension,
+ * and also to simplify handling of holes created by seek past EOF and
+ * consequent write (see buffile.c).
+ */
+void
+decrypt_block(const char *input, char *output, Size size,
+			  const char *key, const char *tweak, bool stream)
+{
+	int			out_size;
+	EVP_CIPHER_CTX *ctx;
+
+	Assert((size >= ENCRYPTION_BLOCK_SIZE &&
+			size % ENCRYPTION_BLOCK_SIZE == 0) || stream);
+
+	/* Ensure encryption has setup */
+	if (!encryption_initialized)
+	{
+		setup_encryption();
+		encryption_initialized = true;
+	}
+
+	if (!stream && IsAllZero(input, size))
+	{
+		memset(output, 0, size);
+		return;
+	}
+
+	ctx = !stream ? ctx_encrypt : ctx_encrypt_stream;
+
+	if (EVP_DecryptInit_ex(ctx, NULL, NULL, (unsigned char *) key,
+						   (unsigned char *) tweak) != 1)
+		evp_error();
+
+	if (EVP_DecryptUpdate(ctx, (unsigned char *) output,
+						  &out_size, (unsigned char *) input, size) != 1)
+		evp_error();
+
+	Assert(out_size == size);
+}
+
+static void
+initialize_encryption_context(EVP_CIPHER_CTX **ctx_p, bool stream)
+{
+	EVP_CIPHER_CTX *ctx;
+	const EVP_CIPHER *cipher;
+	int			block_size;
+
+	cipher = !stream ? EVP_aes_256_cbc() : EVP_aes_256_ctr();
+
+	if ((*ctx_p = EVP_CIPHER_CTX_new()) == NULL)
+		evp_error();
+	ctx = *ctx_p;
+	if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1)
+		evp_error();
+
+	/*
+	 * No padding is needed. For a block cipher, the input block size should
+	 * already be a multiple of ENCRYPTION_BLOCK. For stream cipher, we don't
+	 * need padding anyway. This might save some cycles at the OpenSSL end.
+	 * XXX Is it setting worth when we don't call EVP_DecryptFinal_ex()
+	 * anyway?
+	 */
+	EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+	Assert(EVP_CIPHER_CTX_iv_length(ctx) == ENCRYPTION_TWEAK_SIZE);
+	Assert(EVP_CIPHER_CTX_key_length(ctx) == ENCRYPTION_KEY_SIZE);
+	block_size = EVP_CIPHER_CTX_block_size(ctx);
+#ifdef USE_ASSERT_CHECKING
+	if (!stream)
+		Assert(block_size == ENCRYPTION_BLOCK_SIZE);
+	else
+		Assert(block_size == 1);
+#endif
+
+}
+
+/*
+ * Initialize encryption subsystem for use. Must be called before any
+ * encryptable data is read from or written to data directory.
+ */
+static void
+setup_encryption(void)
+{
+	setup_encryption_openssl();
+	initialize_encryption_context(&ctx_encrypt, false);
+	initialize_encryption_context(&ctx_decrypt, false);
+	initialize_encryption_context(&ctx_encrypt_stream, true);
+	initialize_encryption_context(&ctx_decrypt_stream, true);
+}
+
+static void
+setup_encryption_openssl(void)
+{
+	/*
+	 * Setup OpenSSL.
+	 *
+	 * None of these functions should return a value or raise error.
+	 */
+	ERR_load_crypto_strings();
+	OpenSSL_add_all_algorithms();
+	OPENSSL_config(NULL);
+}
+
+/*
+ * Error callback for openssl.
+ */
+static void
+evp_error(void)
+{
+	ERR_print_errors_fp(stderr);
+#ifndef FRONTEND
+
+	/*
+	 * FATAL is the appropriate level because backend can hardly fix anything
+	 * if encryption / decryption has failed.
+	 *
+	 * XXX Do we yet need EVP_CIPHER_CTX_cleanup() here?
+	 */
+	elog(FATAL, "OpenSSL encountered error during encryption or decryption.");
+#else
+	fprintf(stderr,
+			"OpenSSL encountered error during encryption or decryption.");
+	exit(EXIT_FAILURE);
+#endif							/* FRONTEND */
+}
+
+/*
+ * Xlog is encrypted page at a time. Each xlog page gets a unique tweak via
+ * segment and offset. Unfortunately we can't include timeline because
+ * exitArchiveRecovery() can copy part of the last segment of the old timeline
+ * into the first segment of the new timeline.
+ *
+ * TODO Consider teaching exitArchiveRecovery() to decrypt the copied pages
+ * and encrypt them using a tweak that mentions the new timeline.
+ *
+ * The function is located here rather than some of the xlog*.c modules so
+ * that front-end applications can easily use it too.
+ */
+void
+XLogEncryptionTweak(char *tweak, XLogSegNo segment, uint32 offset)
+{
+	memset(tweak, 0, ENCRYPTION_TWEAK_SIZE);
+	memcpy(tweak, &segment, sizeof(XLogSegNo));
+	memcpy(tweak + sizeof(XLogSegNo), &offset, sizeof(offset));
+}
+
+/*
+ * md files are encrypted block at a time. Tweak will alias higher numbered
+ * forks for huge tables.
+ */
+void
+BufferEncryptionTweak(char *tweak, RelFileNode *relnode, ForkNumber forknum,
+					  BlockNumber blocknum)
+{
+	uint32		fork_and_block = (forknum << 24) ^ blocknum;
+
+	memset(tweak, 0, ENCRYPTION_TWEAK_SIZE);
+	memcpy(tweak, relnode, sizeof(RelFileNode));
+	memcpy(tweak + sizeof(RelFileNode), &fork_and_block, 4);
+}
diff --git a/src/common/string.c b/src/common/string.c
index b01a56c..5ba890d 100644
--- a/src/common/string.c
+++ b/src/common/string.c
@@ -42,6 +42,45 @@ pg_str_endswith(const char *str, const char *end)
 	return strcmp(str, end) == 0;
 }
 
+/*
+ * Helper function to check if a page is completely empty.
+ *
+ * TODO Invent name that is more consistent with that of the other function(s)
+ * in this module.
+ */
+bool
+IsAllZero(const char *input, Size size)
+{
+	const char *pos = input;
+	const char *aligned_start = (char *) MAXALIGN64(input);
+	const char *end = input + size;
+
+	/* Check 1 byte at a time until pos is 8 byte aligned */
+	while (pos < aligned_start)
+		if (*pos++ != 0)
+			return false;
+
+	/*
+	 * Run 8 parallel 8 byte checks in one iteration. On 2016 hardware
+	 * slightly faster than 4 parallel checks.
+	 */
+	while (pos + 8 * sizeof(uint64) <= end)
+	{
+		uint64	   *p = (uint64 *) pos;
+
+		if ((p[0] | p[1] | p[2] | p[3] | p[4] | p[5] | p[6] | p[7]) != 0)
+			return false;
+		pos += 8 * sizeof(uint64);
+	}
+
+	/* Handle unaligned tail. */
+	while (pos < end)
+		if (*pos++ != 0)
+			return false;
+
+	return true;
+}
+
 
 /*
  * strtoint --- just like strtol, but returns int not long
diff --git a/src/include/common/string.h b/src/include/common/string.h
index 77f3133..6128d81 100644
--- a/src/include/common/string.h
+++ b/src/include/common/string.h
@@ -14,5 +14,6 @@ extern bool pg_str_endswith(const char *str, const char *end);
 extern int	strtoint(const char *pg_restrict str, char **pg_restrict endptr,
 					 int base);
 extern void pg_clean_ascii(char *str);
+extern bool IsAllZero(const char *input, Size size);
 
 #endif							/* COMMON_STRING_H */
diff --git a/src/include/storage/encryption.h b/src/include/storage/encryption.h
new file mode 100644
index 0000000..25d665b
--- /dev/null
+++ b/src/include/storage/encryption.h
@@ -0,0 +1,90 @@
+/*-------------------------------------------------------------------------
+ *
+ * encryption.huffer
+ *	  Full database encryption support
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/storage/encryption.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef ENCRYPTION_H
+#define ENCRYPTION_H
+
+#include "access/xlogdefs.h"
+#include "port/pg_crc32c.h"
+#include "storage/smgr.h"
+
+/*
+ * The encrypted data is a series of blocks of size
+ * ENCRYPTION_BLOCK. Currently we use the EVP_aes_256_xts implementation. Make
+ * sure the following constants match if adopting another algorithm.
+ */
+#define ENCRYPTION_BLOCK_SIZE 16
+
+/*
+ * The openssl EVP API refers to a block in terms of padding of the output
+ * chunk. That's the purpose of this constant. However the openssl
+ * implementation of AES XTS still uses the 16-byte block internally, as
+ * defined by ENCRYPTION_BLOCK.
+ */
+#define ENCRYPTION_BLOCK_OPENSSL 1
+
+/*
+ * Encryption key and tweak length for AES-256.
+ */
+#define	ENCRYPTION_KEY_SIZE		32
+#define ENCRYPTION_TWEAK_SIZE	16
+
+/*
+ * If one XLOG record ended and the following one started in the same block,
+ * we'd have to either encrypt and decrypt both records together, or encrypt
+ * (after having zeroed the part of the block occupied by the other record)
+ * and decrypt them separate. Neither approach is compatible with streaming
+ * replication. In the first case we can't ask standby not to decrypt the
+ * first record until the second has been streamed. The second approach would
+ * imply streaming of two different versions of the same block two times.
+ *
+ * We avoid this problem by aligning XLOG records to the encryption block
+ * size. This way no adjacent XLOG records should appear in the same block.
+ *
+ * For similar reasons, the alignment to ENCRYPTION_BLOCK also has to be
+ * applied when storing changes to disk in reorderbuffer.c. Another module
+ * that takes the block into account is buffile.c.
+ *
+ * TODO If the configuration allows walsender to decrypt the XLOG stream
+ * before sending it, adjust this expression so that the additional padding is
+ * not added to XLOG records. (Since the XLOG alignment cannot change without
+ * initdb, the same would apply to the configuration variable that makes
+ * walsender perform the decryption. Does such a variable make sense?)
+ */
+#define DO_ENCRYPTION_BLOCK_ALIGN	data_encrypted
+
+/*
+ * Use TYPEALIGN64 since besides record size we also need to align XLogRecPtr.
+ */
+#define ENCRYPTION_BLOCK_ALIGN(LEN)		TYPEALIGN64(ENCRYPTION_BLOCK, (LEN))
+
+/*
+ * Universal computation of XLOG record alignment.
+ */
+#define XLOG_REC_ALIGN(LEN) MAXALIGN(LEN)
+
+extern PGAlignedBlock encrypt_buf;
+
+extern void EncryptBufferBlock(Oid spcOid, const char *tweak,
+							   const char *input, char *output);
+extern void DecryptBufferBlock(Oid spcOid, const char *tweak,
+							   const char *input, char *output);
+extern void encrypt_block(const char *input, char *output, Size size,
+						  const char *key, const char *tweak, bool stream);
+extern void decrypt_block(const char *input, char *output, Size size,
+						  const char *key, const char *tweak, bool stream);
+extern void XLogEncryptionTweak(char *tweak, XLogSegNo segment,
+					uint32 offset);
+extern void BufferEncryptionTweak(char *tweak, RelFileNode *relnode,
+								  ForkNumber forknum, BlockNumber blocknum);
+
+#endif							/* ENCRYPTION_H */
-- 
1.8.3.1