0002-fix-float-aggregates.patch
text/x-diff
Filename: 0002-fix-float-aggregates.patch
Type: text/x-diff
Part: 1
Patch
Format: unified
Series: patch 0002
| File | + | − |
|---|---|---|
| src/backend/utils/adt/float.c | 35 | 0 |
| src/test/regress/expected/aggregates.out | 10 | 10 |
diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c
index 2101d58674..6a717f19bb 100644
--- a/src/backend/utils/adt/float.c
+++ b/src/backend/utils/adt/float.c
@@ -2925,6 +2925,17 @@ float8_accum(PG_FUNCTION_ARGS)
Sxx = get_float8_nan();
}
}
+ else
+ {
+ /*
+ * At the first input, we normally can leave Sxx as 0. However, if
+ * the first input is Inf or NaN, we'd better force Sxx to NaN;
+ * otherwise we will falsely report variance zero when there are no
+ * more inputs.
+ */
+ if (isnan(newval) || isinf(newval))
+ Sxx = get_float8_nan();
+ }
/*
* If we're invoked as an aggregate, we can cheat and modify our first
@@ -2999,6 +3010,17 @@ float4_accum(PG_FUNCTION_ARGS)
Sxx = get_float8_nan();
}
}
+ else
+ {
+ /*
+ * At the first input, we normally can leave Sxx as 0. However, if
+ * the first input is Inf or NaN, we'd better force Sxx to NaN;
+ * otherwise we will falsely report variance zero when there are no
+ * more inputs.
+ */
+ if (isnan(newval) || isinf(newval))
+ Sxx = get_float8_nan();
+ }
/*
* If we're invoked as an aggregate, we can cheat and modify our first
@@ -3225,6 +3247,19 @@ float8_regr_accum(PG_FUNCTION_ARGS)
Sxy = get_float8_nan();
}
}
+ else
+ {
+ /*
+ * At the first input, we normally can leave Sxx et al as 0. However,
+ * if the first input is Inf or NaN, we'd better force the dependent
+ * sums to NaN; otherwise we will falsely report variance zero when
+ * there are no more inputs.
+ */
+ if (isnan(newvalX) || isinf(newvalX))
+ Sxx = Sxy = get_float8_nan();
+ if (isnan(newvalY) || isinf(newvalY))
+ Syy = Sxy = get_float8_nan();
+ }
/*
* If we're invoked as an aggregate, we can cheat and modify our first
diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out
index 0a6884d382..e4ffa5ee42 100644
--- a/src/test/regress/expected/aggregates.out
+++ b/src/test/regress/expected/aggregates.out
@@ -142,25 +142,25 @@ SELECT stddev_pop(3.0::float8), stddev_samp(4.0::float8);
SELECT var_pop('inf'::float8), var_samp('inf'::float8);
var_pop | var_samp
---------+----------
- 0 |
+ NaN |
(1 row)
SELECT stddev_pop('inf'::float8), stddev_samp('inf'::float8);
stddev_pop | stddev_samp
------------+-------------
- 0 |
+ NaN |
(1 row)
SELECT var_pop('nan'::float8), var_samp('nan'::float8);
var_pop | var_samp
---------+----------
- 0 |
+ NaN |
(1 row)
SELECT stddev_pop('nan'::float8), stddev_samp('nan'::float8);
stddev_pop | stddev_samp
------------+-------------
- 0 |
+ NaN |
(1 row)
SELECT var_pop(1.0::float4), var_samp(2.0::float4);
@@ -178,25 +178,25 @@ SELECT stddev_pop(3.0::float4), stddev_samp(4.0::float4);
SELECT var_pop('inf'::float4), var_samp('inf'::float4);
var_pop | var_samp
---------+----------
- 0 |
+ NaN |
(1 row)
SELECT stddev_pop('inf'::float4), stddev_samp('inf'::float4);
stddev_pop | stddev_samp
------------+-------------
- 0 |
+ NaN |
(1 row)
SELECT var_pop('nan'::float4), var_samp('nan'::float4);
var_pop | var_samp
---------+----------
- 0 |
+ NaN |
(1 row)
SELECT stddev_pop('nan'::float4), stddev_samp('nan'::float4);
stddev_pop | stddev_samp
------------+-------------
- 0 |
+ NaN |
(1 row)
SELECT var_pop(1.0::numeric), var_samp(2.0::numeric);
@@ -393,13 +393,13 @@ SELECT covar_pop(1::float8,2::float8), covar_samp(3::float8,4::float8);
SELECT covar_pop(1::float8,'inf'::float8), covar_samp(3::float8,'inf'::float8);
covar_pop | covar_samp
-----------+------------
- 0 |
+ NaN |
(1 row)
SELECT covar_pop(1::float8,'nan'::float8), covar_samp(3::float8,'nan'::float8);
covar_pop | covar_samp
-----------+------------
- 0 |
+ NaN |
(1 row)
-- test accum and combine functions directly