Re: PATCH: Add uri percent-encoding for binary data

On Mon, Oct  7, 2019 at 09:14:38AM +0200, Anders Åstrand wrote:
> Hello
> 
> Attached is a patch for adding uri as an encoding option for
> encode/decode. It uses what's called "percent-encoding" in rfc3986
> (https://tools.ietf.org/html/rfc3986#section-2.1).

Oh, that's a cool idea.  Can you add it to the commit-fest?

	https://commitfest.postgresql.org/25/

---------------------------------------------------------------------------


> 
> The background for this patch is that I could easily build urls in
> plpgsql, but doing the actual encoding of the url parts is painfully
> slow. The list of available encodings for encode/decode looks quite
> arbitrary to me, so I can't see any reason this one couldn't be in
> there.
> 
> In modern web scenarios one would probably most likely want to encode
> the utf8 representation of a text string for inclusion in a url, in
> which case correct invocation would be ENCODE(CONVERT_TO('some text in
> database encoding goes here', 'UTF8'), 'uri'), but uri
> percent-encoding can of course also be used for other text encodings
> and arbitrary binary data.
> 
> Regards,
> Anders

> diff --git a/src/backend/utils/adt/encode.c b/src/backend/utils/adt/encode.c
> index 7293d66de5..33cf7bb57c 100644
> --- a/src/backend/utils/adt/encode.c
> +++ b/src/backend/utils/adt/encode.c
> @@ -512,6 +512,131 @@ esc_dec_len(const char *src, unsigned srclen)
>  	return len;
>  }
>  
> +/*
> + * URI percent encoding
> + *
> + * Percent encodes all byte values except the unreserved ASCII characters as per RFC3986.
> + */
> +
> +static const char upper_hex_digits[] = "0123456789ABCDEF";
> +
> +static unsigned
> +uri_encode(const char *src, unsigned srclen, char *dst)
> +{
> +	char		*d = dst;
> +
> +	for (const char *s = src; s < src + srclen; s++)
> +	{
> +		if ((*s >= 'A' && *s <= 'Z') ||
> +			(*s >= 'a' && *s <= 'z') ||
> +			(*s >= '0' && *s <= '9') ||
> +			*s == '-' ||
> +			*s == '_' ||
> +			*s == '.' ||
> +			*s == '~')
> +		{
> +			*d++ = *s;
> +		}
> +		else
> +		{
> +			*d++ = '%';
> +			*d++ = upper_hex_digits[(*s >> 4) & 0xF];
> +			*d++ = upper_hex_digits[*s & 0xF];
> +		}
> +	}
> +	return d - dst;
> +}
> +
> +static unsigned
> +uri_decode(const char *src, unsigned srclen, char *dst)
> +{
> +	const char *s = src;
> +	const char *srcend = src + srclen;
> +	char		*d = dst;
> +	char		val;
> +
> +	while (s < srcend)
> +	{
> +		if (*s == '%')
> +		{
> +			if (s > srcend - 3) {
> +				/* This will never get triggered since uri_dec_len already takes care of validation
> +				 */
> +				ereport(ERROR,
> +						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
> +						 errmsg("invalid uri percent encoding"),
> +						 errhint("Input data ends prematurely.")));
> +			}
> +
> +			/* Skip '%' */
> +			s++;
> +
> +			val = get_hex(*s++) << 4;
> +			val += get_hex(*s++);
> +			*d++ = val;
> +		}
> +		else
> +		{
> +			*d++ = *s++;
> +		}
> +	}
> +	return d - dst;
> +}
> +
> +static unsigned
> +uri_enc_len(const char *src, unsigned srclen)
> +{
> +	int			len = 0;
> +
> +	for (const char *s = src; s < src + srclen; s++)
> +	{
> +		if ((*s >= 'A' && *s <= 'Z') ||
> +			(*s >= 'a' && *s <= 'z') ||
> +			(*s >= '0' && *s <= '9') ||
> +			*s == '-' ||
> +			*s == '_' ||
> +			*s == '.' ||
> +			*s == '~')
> +		{
> +			len++;
> +		}
> +		else
> +		{
> +			len += 3;
> +		}
> +	}
> +	return len;
> +}
> +
> +static unsigned
> +uri_dec_len(const char *src, unsigned srclen)
> +{
> +	const char *s = src;
> +	const char *srcend = src + srclen;
> +	int			len = 0;
> +
> +	while (s < srcend)
> +	{
> +		if (*s == '%')
> +		{
> +			if (s > srcend - 3) {
> +				ereport(ERROR,
> +						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
> +						 errmsg("invalid uri percent encoding"),
> +						 errhint("Input data ends prematurely.")));
> +			}
> +			s++;
> +			get_hex(*s++);
> +			get_hex(*s++);
> +		}
> +		else {
> +			s++;
> +		}
> +		len++;
> +	}
> +	return len;
> +}
> +
>  /*
>   * Common
>   */
> @@ -541,6 +666,12 @@ static const struct
>  			esc_enc_len, esc_dec_len, esc_encode, esc_decode
>  		}
>  	},
> +	{
> +		"uri",
> +		{
> +			uri_enc_len, uri_dec_len, uri_encode, uri_decode
> +		}
> +	},
>  	{
>  		NULL,
>  		{
> diff --git a/src/test/regress/expected/strings.out b/src/test/regress/expected/strings.out
> index 2483966576..f89c5ec1c3 100644
> --- a/src/test/regress/expected/strings.out
> +++ b/src/test/regress/expected/strings.out
> @@ -1870,3 +1870,24 @@ SELECT encode(overlay(E'Th\\000omas'::bytea placing E'\\002\\003'::bytea from 5
>   Th\000o\x02\x03
>  (1 row)
>  
> +SET bytea_output TO hex;
> +SELECT encode(E'en\\300\\336d'::bytea, 'uri');
> +  encode   
> +-----------
> + en%C0%DEd
> +(1 row)
> +
> +SELECT decode('%De%c0%DEd', 'uri');
> +   decode   
> +------------
> + \xdec0de64
> +(1 row)
> +
> +SELECT decode('error%Ex', 'uri');
> +ERROR:  invalid hexadecimal digit: "x"
> +SELECT decode('error%E', 'uri');
> +ERROR:  invalid uri percent encoding
> +HINT:  Input data ends prematurely.
> +SELECT decode('error%', 'uri');
> +ERROR:  invalid uri percent encoding
> +HINT:  Input data ends prematurely.
> diff --git a/src/test/regress/sql/strings.sql b/src/test/regress/sql/strings.sql
> index b5e75c344f..1d03836b6e 100644
> --- a/src/test/regress/sql/strings.sql
> +++ b/src/test/regress/sql/strings.sql
> @@ -641,3 +641,10 @@ SELECT btrim(E'\\000trim\\000'::bytea, ''::bytea);
>  SELECT encode(overlay(E'Th\\000omas'::bytea placing E'Th\\001omas'::bytea from 2),'escape');
>  SELECT encode(overlay(E'Th\\000omas'::bytea placing E'\\002\\003'::bytea from 8),'escape');
>  SELECT encode(overlay(E'Th\\000omas'::bytea placing E'\\002\\003'::bytea from 5 for 3),'escape');
> +
> +SET bytea_output TO hex;
> +SELECT encode(E'en\\300\\336d'::bytea, 'uri');
> +SELECT decode('%De%c0%DEd', 'uri');
> +SELECT decode('error%Ex', 'uri');
> +SELECT decode('error%E', 'uri');
> +SELECT decode('error%', 'uri');


-- 
  Bruce Momjian  <bruce@momjian.us>        http://momjian.us
  EnterpriseDB                             http://enterprisedb.com

+ As you are, so once was I.  As I am, so you will be. +
+                      Ancient Roman grave inscription +