From 65dbe35271a3800037858058dcec77b186e0df05 Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Fri, 21 Feb 2025 14:20:18 +0100 Subject: [PATCH v7b 3/3] Add pg_shmem_numa_allocations to show NUMA zones for shared memory allocations Author: Jakub Wartak Co-authored-by: Bertrand Drouvot Reviewed-by: Andres Freund Discussion: https://postgr.es/m/CAKZiRmxh6KWo0aqRqvmcoaX2jUxZYb4kGp3N%3Dq1w%2BDiH-696Xw%40mail.gmail.com --- src/backend/catalog/system_views.sql | 8 ++ src/backend/storage/ipc/shmem.c | 125 +++++++++++++++++++++++ src/include/catalog/pg_proc.dat | 8 ++ src/test/regress/expected/privileges.out | 25 ++++- src/test/regress/expected/rules.out | 4 + src/test/regress/sql/privileges.sql | 10 +- 6 files changed, 176 insertions(+), 4 deletions(-) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index a4d2cfdcaf..cc014a62dc 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -658,6 +658,14 @@ GRANT SELECT ON pg_shmem_allocations TO pg_read_all_stats; REVOKE EXECUTE ON FUNCTION pg_get_shmem_allocations() FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_get_shmem_allocations() TO pg_read_all_stats; +CREATE VIEW pg_shmem_numa_allocations AS + SELECT * FROM pg_get_shmem_numa_allocations(); + +REVOKE ALL ON pg_shmem_numa_allocations FROM PUBLIC; +GRANT SELECT ON pg_shmem_numa_allocations TO pg_read_all_stats; +REVOKE EXECUTE ON FUNCTION pg_get_shmem_numa_allocations() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_shmem_numa_allocations() TO pg_read_all_stats; + CREATE VIEW pg_backend_memory_contexts AS SELECT * FROM pg_get_backend_memory_contexts(); diff --git a/src/backend/storage/ipc/shmem.c b/src/backend/storage/ipc/shmem.c index 895a43fb39..7d83a14390 100644 --- a/src/backend/storage/ipc/shmem.c +++ b/src/backend/storage/ipc/shmem.c @@ -68,6 +68,7 @@ #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" +#include "port/pg_numa.h" #include "storage/lwlock.h" #include "storage/pg_shmem.h" #include "storage/shmem.h" @@ -568,3 +569,127 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS) return (Datum) 0; } + +/* SQL SRF showing NUMA zones for allocated shared memory */ +Datum +pg_get_shmem_numa_allocations(PG_FUNCTION_ARGS) +{ +#define PG_GET_SHMEM_NUMA_SIZES_COLS 3 + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + HASH_SEQ_STATUS hstat; + ShmemIndexEnt *ent; + Datum values[PG_GET_SHMEM_NUMA_SIZES_COLS]; + bool nulls[PG_GET_SHMEM_NUMA_SIZES_COLS]; + Size os_page_size; + void **page_ptrs; + int *pages_status; + int shm_total_page_count, + shm_ent_page_count; + static bool firstUseInBackend = true; + + InitMaterializedSRF(fcinfo, 0); + + if (pg_numa_init() == -1) + { + elog(NOTICE, "libnuma initialization failed or NUMA is not supported on this platform, some NUMA data might be unavailable.");; + return (Datum) 0; + } + + /* + * This is for gathering some NUMA statistics. We might be using various + * DB block sizes (4kB, 8kB , .. 32kB) that end up being allocated in + * various different OS memory pages sizes, so first we need to understand + * the OS memory page size before calling move_pages() + */ + os_page_size = pg_numa_get_pagesize(); + + /* + * Preallocate memory all at once without going into details which shared + * memory segment is the biggest (technically min s_b can be as low as + * 16xBLCKSZ) + */ + shm_total_page_count = ShmemSegHdr->totalsize / os_page_size; + page_ptrs = palloc(sizeof(void *) * shm_total_page_count); + pages_status = palloc(sizeof(int) * shm_total_page_count); + memset(page_ptrs, 0, sizeof(void *) * shm_total_page_count); + + if (firstUseInBackend == true) + elog(DEBUG1, "NUMA: page-faulting shared memory segments for proper NUMA readouts"); + + LWLockAcquire(ShmemIndexLock, LW_SHARED); + + hash_seq_init(&hstat, ShmemIndex); + + /* output all allocated entries */ + memset(nulls, 0, sizeof(nulls)); + while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL) + { + int i; +#define MAX_NUMA_ZONES 32 /* FIXME? */ + Size zones[MAX_NUMA_ZONES]; + + shm_ent_page_count = ent->allocated_size / os_page_size; + /* It is always at least 1 page */ + shm_ent_page_count = shm_ent_page_count == 0 ? 1 : shm_ent_page_count; + + /* + * If we get ever 0xff back from kernel inquiry, then we probably have + * bug in our buffers to OS page mapping code here + */ + memset(pages_status, 0xff, sizeof(int) * shm_ent_page_count); + + for (i = 0; i < shm_ent_page_count; i++) + { + /* + * In order to get reliable results we also need to touch memory + * pages so that inquiry about NUMA zone doesn't return -2. + */ + volatile uint64 touch pg_attribute_unused(); + + page_ptrs[i] = (char *) ent->location + (i * os_page_size); + if (firstUseInBackend == true) + pg_numa_touch_mem_if_required(touch, page_ptrs[i]); + + CHECK_FOR_INTERRUPTS(); + } + + if (pg_numa_query_pages(0, shm_ent_page_count, page_ptrs, pages_status) == -1) + { + /* FIXME: should we release LWlock here ? */ + elog(ERROR, "failed NUMA pages inquiry status: %m"); + } + + memset(zones, 0, sizeof(zones)); + /* Count number of NUMA zones used for this shared memory entry */ + for (i = 0; i < shm_ent_page_count; i++) + { + int s = pages_status[i]; + + if (s >= 0) + zones[s]++; + } + + for (i = 0; i <= pg_numa_get_max_node(); i++) + { + values[0] = CStringGetTextDatum(ent->key); + values[1] = i; + values[2] = Int64GetDatum(zones[i] * os_page_size); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + + } + + /* + * XXX: We are ignoring in NUMA version reporting of the following regions + * (compare to pg_get_shmem_allocations() case): 1. output shared memory + * allocated but not counted via the shmem index 2. output as-of-yet + * unused shared memory + */ + + LWLockRelease(ShmemIndexLock); + firstUseInBackend = false; + + return (Datum) 0; +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index cd9422d0ba..62f051c194 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -8463,6 +8463,14 @@ proargnames => '{name,off,size,allocated_size}', prosrc => 'pg_get_shmem_allocations' }, +# shared memory usage with NUMA info +{ oid => '5101', descr => 'NUMA mappings for the main shared memory segment', + proname => 'pg_get_shmem_numa_allocations', prorows => '50', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => '', + proallargtypes => '{text,int8,int8}', proargmodes => '{o,o,o}', + proargnames => '{name,numa_zone_id,numa_size}', + prosrc => 'pg_get_shmem_numa_allocations' }, + # memory context of local backend { oid => '2282', descr => 'information about all memory contexts of local backend', diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index a76256405f..02997690e1 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -3127,8 +3127,8 @@ REVOKE MAINTAIN ON lock_table FROM regress_locktable_user; -- clean up DROP TABLE lock_table; DROP USER regress_locktable_user; --- test to check privileges of system views pg_shmem_allocations and --- pg_backend_memory_contexts. +-- test to check privileges of system views pg_shmem_allocations, +-- pg_shmem_numa_allocations and pg_backend_memory_contexts. -- switch to superuser \c - CREATE ROLE regress_readallstats; @@ -3144,6 +3144,12 @@ SELECT has_table_privilege('regress_readallstats','pg_shmem_allocations','SELECT f (1 row) +SELECT has_table_privilege('regress_readallstats','pg_shmem_numa_allocations','SELECT'); -- no + has_table_privilege +--------------------- + f +(1 row) + GRANT pg_read_all_stats TO regress_readallstats; SELECT has_table_privilege('regress_readallstats','pg_backend_memory_contexts','SELECT'); -- yes has_table_privilege @@ -3157,6 +3163,12 @@ SELECT has_table_privilege('regress_readallstats','pg_shmem_allocations','SELECT t (1 row) +SELECT has_table_privilege('regress_readallstats','pg_shmem_numa_allocations','SELECT'); -- yes + has_table_privilege +--------------------- + t +(1 row) + -- run query to ensure that functions within views can be executed SET ROLE regress_readallstats; SELECT COUNT(*) >= 0 AS ok FROM pg_backend_memory_contexts; @@ -3171,6 +3183,15 @@ SELECT COUNT(*) >= 0 AS ok FROM pg_shmem_allocations; t (1 row) +-- to ignore potenital NOTICE: libnuma initialization failed.. +SET client_min_messages TO warning ; +SELECT COUNT(*) >= 0 AS ok FROM pg_shmem_numa_allocations; + ok +---- + t +(1 row) + +RESET client_min_messages; RESET ROLE; -- clean up DROP ROLE regress_readallstats; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 62f69ac20b..b63c6e0f74 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1740,6 +1740,10 @@ pg_shmem_allocations| SELECT name, size, allocated_size FROM pg_get_shmem_allocations() pg_get_shmem_allocations(name, off, size, allocated_size); +pg_shmem_numa_allocations| SELECT name, + numa_zone_id, + numa_size + FROM pg_get_shmem_numa_allocations() pg_get_shmem_numa_allocations(name, numa_zone_id, numa_size); pg_stat_activity| SELECT s.datid, d.datname, s.pid, diff --git a/src/test/regress/sql/privileges.sql b/src/test/regress/sql/privileges.sql index d195aaf137..e969cc3854 100644 --- a/src/test/regress/sql/privileges.sql +++ b/src/test/regress/sql/privileges.sql @@ -1911,8 +1911,8 @@ REVOKE MAINTAIN ON lock_table FROM regress_locktable_user; DROP TABLE lock_table; DROP USER regress_locktable_user; --- test to check privileges of system views pg_shmem_allocations and --- pg_backend_memory_contexts. +-- test to check privileges of system views pg_shmem_allocations, +-- pg_shmem_numa_allocations and pg_backend_memory_contexts. -- switch to superuser \c - @@ -1921,16 +1921,22 @@ CREATE ROLE regress_readallstats; SELECT has_table_privilege('regress_readallstats','pg_backend_memory_contexts','SELECT'); -- no SELECT has_table_privilege('regress_readallstats','pg_shmem_allocations','SELECT'); -- no +SELECT has_table_privilege('regress_readallstats','pg_shmem_numa_allocations','SELECT'); -- no GRANT pg_read_all_stats TO regress_readallstats; SELECT has_table_privilege('regress_readallstats','pg_backend_memory_contexts','SELECT'); -- yes SELECT has_table_privilege('regress_readallstats','pg_shmem_allocations','SELECT'); -- yes +SELECT has_table_privilege('regress_readallstats','pg_shmem_numa_allocations','SELECT'); -- yes -- run query to ensure that functions within views can be executed SET ROLE regress_readallstats; SELECT COUNT(*) >= 0 AS ok FROM pg_backend_memory_contexts; SELECT COUNT(*) >= 0 AS ok FROM pg_shmem_allocations; +-- to ignore potenital NOTICE: libnuma initialization failed.. +SET client_min_messages TO warning ; +SELECT COUNT(*) >= 0 AS ok FROM pg_shmem_numa_allocations; +RESET client_min_messages; RESET ROLE; -- clean up -- 2.39.5