*** ./contrib/dict_preload/dict_preload.c.orig 2010-03-18 17:00:33.000000000 +0100 --- ./contrib/dict_preload/dict_preload.c 2010-03-31 19:45:51.123855008 +0200 *************** *** 0 **** --- 1,183 ---- + /*------------------------------------------------------------------------- + * + * dict_preload.c + * preloaded dictionary + * + * Copyright (c) 2007-2010, PostgreSQL Global Development Group + * + * IDENTIFICATION + * $PostgreSQL: pgsql/contrib/dict_preload/dict_preload.c,v 1.6 2010/01/02 16:57:32 momjian Exp $ + * + *------------------------------------------------------------------------- + */ + #include "postgres.h" + + #include "commands/defrem.h" + #include "fmgr.h" + #include "miscadmin.h" + #include "nodes/makefuncs.h" + #include "nodes/value.h" + #include "tsearch/ts_public.h" + #include "tsearch/ts_utils.h" + #include "tsearch/dicts/spell.h" + #include "utils/guc.h" + #include "utils/memutils.h" + + PG_MODULE_MAGIC; + + char *preload_dictfile = NULL; + char *preload_afffile = NULL; + char *preload_stopwords = NULL; + + typedef struct + { + StopList stoplist; + IspellDict obj; + } DictISpell; + + MemoryContext preload_ctx = NULL; + + DictISpell *preload_dict = NULL; + + extern MemoryContext MMapAllocSetContextCreate(MemoryContext parent, + const char *name, + Size minContextSize, + Size initBlockSize, + Size maxBlockSize); + + PG_FUNCTION_INFO_V1(dpreloaddict_init); + Datum dpreloaddict_init(PG_FUNCTION_ARGS); + + PG_FUNCTION_INFO_V1(dpreloaddict_lexize); + Datum dpreloaddict_lexize(PG_FUNCTION_ARGS); + + void _PG_init(void); + void _PG_fini(void); + + static DictISpell * + load_dictionary(void) + { + List *dictopt = NIL; + FunctionCallInfoData fcinfo; + DictISpell *result; + + /* + * read parameters for preloaded dictionary + */ + if (preload_dictfile != NULL) + dictopt = lappend(dictopt, makeDefElem("DictFile", + (Node *) makeString(preload_dictfile))); + if (preload_afffile != NULL) + dictopt = lappend(dictopt, makeDefElem("AffFile", + (Node *) makeString(preload_afffile))); + if (preload_stopwords != NULL) + dictopt = lappend(dictopt, makeDefElem("StopWords", + (Node *) makeString(preload_stopwords))); + /* + * Initialise ispell dictionary + */ + InitFunctionCallInfoData(fcinfo, NULL, 1, NULL, NULL); + fcinfo.arg[0] = PointerGetDatum(dictopt); + fcinfo.argnull[0] = false; + + result = (DictISpell *) DatumGetPointer(dispell_init(&fcinfo)); + + /* + * MemoryContextStats(CurrentMemoryContext); + */ + + return result; + } + + Datum + dpreloaddict_init(PG_FUNCTION_ARGS) + { + static bool firsttime = true; + + /* + * dpreloaddict_init can be called more times: + * CREATE TEXT SEARCH DICTIONARY + * DROP TEXT SEARCH DICTIONARY + * CREATE TEXT SEARCH DICTIONARY + * ... + */ + if (firsttime) + { + /* In this moment, dictionary have to be loaded */ + Assert(MemoryContextIsValid(preload_ctx)); + Assert(preload_dict != NULL); + + preload_ctx = NULL; + firsttime = false; + + return PointerGetDatum(preload_dict); + } + else + return PointerGetDatum(load_dictionary()); + } + + Datum + dpreloaddict_lexize(PG_FUNCTION_ARGS) + { + return dispell_lexize(fcinfo); + } + + /* + * Module load callback + */ + void + _PG_init() + { + MemoryContext oldctx; + static bool inited = false; + GucContext guc_ctx; + + if (inited) + return; + else + inited = true; + + guc_ctx = process_shared_preload_libraries_in_progress ? + PGC_POSTMASTER : PGC_SUSET; + + /* Define custom GUC variables. */ + DefineCustomStringVariable("dict_preload.dictfile", + "name of file of preloaded ispell dictionary", + NULL, + &preload_dictfile, + NULL, + guc_ctx, 0, + NULL, NULL); + + /* Define custom GUC variables. */ + DefineCustomStringVariable("dict_preload.afffile", + "name of file of preloaded ispell affix", + NULL, + &preload_afffile, + NULL, + guc_ctx, 0, + NULL, NULL); + + /* Define custom GUC variables. */ + DefineCustomStringVariable("dict_preload.stopwords", + "name of file of preloaded ispell stopwords", + NULL, + &preload_stopwords, + NULL, + guc_ctx, 0, + NULL, NULL); + + /* preload dictionary */ + Assert(preload_ctx == NULL); + Assert(preload_dict == NULL); + + preload_ctx = MMapAllocSetContextCreate(NULL, "Ispell dictionary preload context", + 512 * 1024, + 1024 * 1024 *4, + 1024 * 1024 *8); + oldctx = MemoryContextSwitchTo(preload_ctx); + + preload_dict = load_dictionary(); + + MemoryContextSwitchTo(oldctx); + } *** ./contrib/dict_preload/dict_preload.sql.in.orig 2010-03-18 17:00:52.000000000 +0100 --- ./contrib/dict_preload/dict_preload.sql.in 2010-03-19 08:24:33.000000000 +0100 *************** *** 0 **** --- 1,19 ---- + /* $PostgreSQL: pgsql/contrib/dict_int/dict_int.sql.in,v 1.3 2007/11/13 04:24:27 momjian Exp $ */ + + -- Adjust this setting to control where the objects get created. + SET search_path = public; + + CREATE OR REPLACE FUNCTION dpreloaddict_init(internal) + RETURNS internal + AS 'MODULE_PATHNAME' + LANGUAGE C STRICT; + + CREATE OR REPLACE FUNCTION dpreloaddict_lexize(internal, internal, internal, internal) + RETURNS internal + AS 'MODULE_PATHNAME' + LANGUAGE C STRICT; + + CREATE TEXT SEARCH TEMPLATE preloaddict( + LEXIZE = dpreloaddict_lexize, + INIT = dpreloaddict_init + ); *** ./contrib/dict_preload/mmap_alloc.c.orig 2010-03-30 16:52:16.034361127 +0200 --- ./contrib/dict_preload/mmap_alloc.c 2010-03-31 19:44:56.428854686 +0200 *************** *** 0 **** --- 1,1300 ---- + /*------------------------------------------------------------------------- + * + * aset.c + * Allocation set definitions. + * + * AllocSet is our standard implementation of the abstract MemoryContext + * type. + * + * + * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * $PostgreSQL: pgsql/src/backend/utils/mmgr/aset.c,v 1.83 2010/02/26 02:01:14 momjian Exp $ + * + * NOTE: + * This is a new (Feb. 05, 1999) implementation of the allocation set + * routines. AllocSet...() does not use OrderedSet...() any more. + * Instead it manages allocations in a block pool by itself, combining + * many small allocations in a few bigger blocks. AllocSetFree() normally + * doesn't free() memory really. It just add's the free'd area to some + * list for later reuse by AllocSetAlloc(). All memory blocks are free()'d + * at once on AllocSetReset(), which happens when the memory context gets + * destroyed. + * Jan Wieck + * + * Performance improvement from Tom Lane, 8/99: for extremely large request + * sizes, we do want to be able to give the memory back to free() as soon + * as it is pfree()'d. Otherwise we risk tying up a lot of memory in + * freelist entries that might never be usable. This is specially needed + * when the caller is repeatedly repalloc()'ing a block bigger and bigger; + * the previous instances of the block were guaranteed to be wasted until + * AllocSetReset() under the old way. + * + * Further improvement 12/00: as the code stood, request sizes in the + * midrange between "small" and "large" were handled very inefficiently, + * because any sufficiently large free chunk would be used to satisfy a + * request, even if it was much larger than necessary. This led to more + * and more wasted space in allocated chunks over time. To fix, get rid + * of the midrange behavior: we now handle only "small" power-of-2-size + * chunks as chunks. Anything "large" is passed off to malloc(). Change + * the number of freelists to change the small/large boundary. + * + * + * About CLOBBER_FREED_MEMORY: + * + * If this symbol is defined, all freed memory is overwritten with 0x7F's. + * This is useful for catching places that reference already-freed memory. + * + * About MEMORY_CONTEXT_CHECKING: + * + * Since we usually round request sizes up to the next power of 2, there + * is often some unused space immediately after a requested data area. + * Thus, if someone makes the common error of writing past what they've + * requested, the problem is likely to go unnoticed ... until the day when + * there *isn't* any wasted space, perhaps because of different memory + * alignment on a new platform, or some other effect. To catch this sort + * of problem, the MEMORY_CONTEXT_CHECKING option stores 0x7E just beyond + * the requested space whenever the request is less than the actual chunk + * size, and verifies that the byte is undamaged when the chunk is freed. + * + *------------------------------------------------------------------------- + */ + + #include + #include + #include + + #include "postgres.h" + + #include "utils/memutils.h" + + /* Define this to detail debug alloc information */ + /* #define HAVE_ALLOCINFO */ + + /*-------------------- + * Chunk freelist k holds chunks of size 1 << (k + ALLOC_MINBITS), + * for k = 0 .. ALLOCSET_NUM_FREELISTS-1. + * + * Note that all chunks in the freelists have power-of-2 sizes. This + * improves recyclability: we may waste some space, but the wasted space + * should stay pretty constant as requests are made and released. + * + * A request too large for the last freelist is handled by allocating a + * dedicated block from malloc(). The block still has a block header and + * chunk header, but when the chunk is freed we'll return the whole block + * to malloc(), not put it on our freelists. + * + * CAUTION: ALLOC_MINBITS must be large enough so that + * 1<header.name, (_chunk), (_chunk)->size) + #define AllocAllocInfo(_cxt, _chunk) \ + fprintf(stderr, "AllocAlloc: %s: %p, %d\n", \ + (_cxt)->header.name, (_chunk), (_chunk)->size) + #else + #define AllocFreeInfo(_cxt, _chunk) + #define AllocAllocInfo(_cxt, _chunk) + #endif + + #define PAGEALIGN(size) (((size) / getpagesize() + 1) * getpagesize()) + + static void * + mmap_alloc(Size size) + { + void *ptr; + + size = PAGEALIGN(MAXALIGN(size) + MAXALIGN(sizeof(Size))); + + ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANON, + -1, 0); + if (ptr == MAP_FAILED) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) size))); + + *((Size*) ptr) = size; + return (char *) ptr + MAXALIGN(sizeof(Size)); + } + + static void + mmap_free(void *ptr) + { + Size size; + + ptr = (char *) ptr - MAXALIGN(sizeof(Size)); + size = *((Size*) ptr); + + munmap(ptr, size); + } + + static void * + mmap_realloc(void *ptr, Size size) + { + Size oldsize; + void *result = ptr; + int i; + char *x; + + ptr = (char *) ptr - MAXALIGN(sizeof(Size)); + + oldsize = *((Size *) ptr); + size = PAGEALIGN(MAXALIGN(size) + MAXALIGN(sizeof(Size))); + + /* + * Attention - on linux you cannot call mremap together with + * shared + * + * ptr = mremap(ptr, oldsize, + * size, + * MREMAP_MAYMOVE); + */ + if (size > oldsize) + { + void *newptr; + Size newsize = PAGEALIGN(2 * oldsize); + + while (newsize < size) + newsize = PAGEALIGN(2 * newsize); + + newptr = mmap(NULL, newsize, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANON, + -1, 0); + + if (newptr == MAP_FAILED) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) newsize))); + memcpy(newptr, ptr, oldsize); + *((Size*) newptr) = newsize; + + munmap(ptr, oldsize); + result = (char *) newptr + MAXALIGN(sizeof(Size)); + } + + return result; + } + + /* ---------- + * AllocSetFreeIndex - + * + * Depending on the size of an allocation compute which freechunk + * list of the alloc set it belongs to. Caller must have verified + * that size <= ALLOC_CHUNK_LIMIT. + * ---------- + */ + static inline int + AllocSetFreeIndex(Size size) + { + int idx; + unsigned int t, + tsize; + + if (size > (1 << ALLOC_MINBITS)) + { + tsize = (size - 1) >> ALLOC_MINBITS; + + /* + * At this point we need to obtain log2(tsize)+1, ie, the number of + * not-all-zero bits at the right. We used to do this with a + * shift-and-count loop, but this function is enough of a hotspot to + * justify micro-optimization effort. The best approach seems to be + * to use a lookup table. Note that this code assumes that + * ALLOCSET_NUM_FREELISTS <= 17, since we only cope with two bytes of + * the tsize value. + */ + t = tsize >> 8; + idx = t ? LogTable256[t] + 8 : LogTable256[tsize]; + + Assert(idx < ALLOCSET_NUM_FREELISTS); + } + else + idx = 0; + + return idx; + } + + #ifdef RANDOMIZE_ALLOCATED_MEMORY + + /* + * Fill a just-allocated piece of memory with "random" data. It's not really + * very random, just a repeating sequence with a length that's prime. What + * we mainly want out of it is to have a good probability that two palloc's + * of the same number of bytes start out containing different data. + */ + static void + randomize_mem(char *ptr, size_t size) + { + static int save_ctr = 1; + int ctr; + + ctr = save_ctr; + while (size-- > 0) + { + *ptr++ = ctr; + if (++ctr > 251) + ctr = 1; + } + save_ctr = ctr; + } + #endif /* RANDOMIZE_ALLOCATED_MEMORY */ + + + /* + * Public routines + */ + + + /* + * AllocSetContextCreate + * Create a new AllocSet context. + * + * parent: parent context, or NULL if top-level context + * name: name of context (for debugging --- string will be copied) + * minContextSize: minimum context size + * initBlockSize: initial allocation block size + * maxBlockSize: maximum allocation block size + */ + MemoryContext + MMapAllocSetContextCreate(MemoryContext parent, + const char *name, + Size minContextSize, + Size initBlockSize, + Size maxBlockSize) + { + AllocSet context; + + /* Do the type-independent part of context creation */ + context = (AllocSet) MemoryContextCreate(T_AllocSetContext, + sizeof(AllocSetContext), + &AllocSetMethods, + parent, + name); + + /* + * Make sure alloc parameters are reasonable, and save them. + * + * We somewhat arbitrarily enforce a minimum 1K block size. + */ + initBlockSize = MAXALIGN(initBlockSize); + if (initBlockSize < 1024) + initBlockSize = 1024; + maxBlockSize = MAXALIGN(maxBlockSize); + if (maxBlockSize < initBlockSize) + maxBlockSize = initBlockSize; + context->initBlockSize = initBlockSize; + context->maxBlockSize = maxBlockSize; + context->nextBlockSize = initBlockSize; + + /* + * Compute the allocation chunk size limit for this context. It can't be + * more than ALLOC_CHUNK_LIMIT because of the fixed number of freelists. + * If maxBlockSize is small then requests exceeding the maxBlockSize + * should be treated as large chunks, too. We have to have + * allocChunkLimit a power of two, because the requested and + * actually-allocated sizes of any chunk must be on the same side of the + * limit, else we get confused about whether the chunk is "big". + */ + context->allocChunkLimit = ALLOC_CHUNK_LIMIT; + while (context->allocChunkLimit > + (Size) (maxBlockSize - ALLOC_BLOCKHDRSZ - ALLOC_CHUNKHDRSZ)) + context->allocChunkLimit >>= 1; + + /* + * Grab always-allocated space, if requested + */ + if (minContextSize > ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ) + { + Size blksize = MAXALIGN(minContextSize); + AllocBlock block; + + block = (AllocBlock) mmap_alloc(blksize); + if (block == NULL) + { + MemoryContextStats(TopMemoryContext); + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed while creating memory context \"%s\".", + name))); + } + block->aset = context; + block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; + block->endptr = ((char *) block) + blksize; + block->next = context->blocks; + context->blocks = block; + /* Mark block as not to be released at reset time */ + context->keeper = block; + } + + context->isReset = true; + + return (MemoryContext) context; + } + + /* + * AllocSetInit + * Context-type-specific initialization routine. + * + * This is called by MemoryContextCreate() after setting up the + * generic MemoryContext fields and before linking the new context + * into the context tree. We must do whatever is needed to make the + * new context minimally valid for deletion. We must *not* risk + * failure --- thus, for example, allocating more memory is not cool. + * (AllocSetContextCreate can allocate memory when it gets control + * back, however.) + */ + static void + AllocSetInit(MemoryContext context) + { + /* + * Since MemoryContextCreate already zeroed the context node, we don't + * have to do anything here: it's already OK. + */ + } + + /* + * AllocSetReset + * Frees all memory which is allocated in the given set. + * + * Actually, this routine has some discretion about what to do. + * It should mark all allocated chunks freed, but it need not necessarily + * give back all the resources the set owns. Our actual implementation is + * that we hang onto any "keeper" block specified for the set. In this way, + * we don't thrash malloc() when a context is repeatedly reset after small + * allocations, which is typical behavior for per-tuple contexts. + */ + static void + AllocSetReset(MemoryContext context) + { + AllocSet set = (AllocSet) context; + AllocBlock block; + + AssertArg(AllocSetIsValid(set)); + + /* Nothing to do if no pallocs since startup or last reset */ + if (set->isReset) + return; + + #ifdef MEMORY_CONTEXT_CHECKING + /* Check for corruption and leaks before freeing */ + AllocSetCheck(context); + #endif + + /* Clear chunk freelists */ + MemSetAligned(set->freelist, 0, sizeof(set->freelist)); + + block = set->blocks; + + /* New blocks list is either empty or just the keeper block */ + set->blocks = set->keeper; + + while (block != NULL) + { + AllocBlock next = block->next; + + if (block == set->keeper) + { + /* Reset the block, but don't return it to malloc */ + char *datastart = ((char *) block) + ALLOC_BLOCKHDRSZ; + + #ifdef CLOBBER_FREED_MEMORY + /* Wipe freed memory for debugging purposes */ + memset(datastart, 0x7F, block->freeptr - datastart); + #endif + block->freeptr = datastart; + block->next = NULL; + } + else + { + /* Normal case, release the block */ + #ifdef CLOBBER_FREED_MEMORY + /* Wipe freed memory for debugging purposes */ + memset(block, 0x7F, block->freeptr - ((char *) block)); + #endif + mmap_free(block); + } + block = next; + } + + /* Reset block size allocation sequence, too */ + set->nextBlockSize = set->initBlockSize; + + set->isReset = true; + } + + /* + * AllocSetDelete + * Frees all memory which is allocated in the given set, + * in preparation for deletion of the set. + * + * Unlike AllocSetReset, this *must* free all resources of the set. + * But note we are not responsible for deleting the context node itself. + */ + static void + AllocSetDelete(MemoryContext context) + { + AllocSet set = (AllocSet) context; + AllocBlock block = set->blocks; + + AssertArg(AllocSetIsValid(set)); + + #ifdef MEMORY_CONTEXT_CHECKING + /* Check for corruption and leaks before freeing */ + AllocSetCheck(context); + #endif + + /* Make it look empty, just in case... */ + MemSetAligned(set->freelist, 0, sizeof(set->freelist)); + set->blocks = NULL; + set->keeper = NULL; + + while (block != NULL) + { + AllocBlock next = block->next; + + #ifdef CLOBBER_FREED_MEMORY + /* Wipe freed memory for debugging purposes */ + memset(block, 0x7F, block->freeptr - ((char *) block)); + #endif + mmap_free(block); + block = next; + } + } + + /* + * AllocSetAlloc + * Returns pointer to allocated memory of given size; memory is added + * to the set. + */ + static void * + AllocSetAlloc(MemoryContext context, Size size) + { + AllocSet set = (AllocSet) context; + AllocBlock block; + AllocChunk chunk; + int fidx; + Size chunk_size; + Size blksize; + + AssertArg(AllocSetIsValid(set)); + + /* + * If requested size exceeds maximum for chunks, allocate an entire block + * for this request. + */ + if (size > set->allocChunkLimit) + { + chunk_size = MAXALIGN(size); + blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; + block = (AllocBlock) mmap_alloc(blksize); + if (block == NULL) + { + MemoryContextStats(TopMemoryContext); + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) size))); + } + block->aset = set; + block->freeptr = block->endptr = ((char *) block) + blksize; + + chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ); + chunk->aset = set; + chunk->size = chunk_size; + #ifdef MEMORY_CONTEXT_CHECKING + chunk->requested_size = size; + /* set mark to catch clobber of "unused" space */ + if (size < chunk_size) + ((char *) AllocChunkGetPointer(chunk))[size] = 0x7E; + #endif + #ifdef RANDOMIZE_ALLOCATED_MEMORY + /* fill the allocated space with junk */ + randomize_mem((char *) AllocChunkGetPointer(chunk), size); + #endif + + /* + * Stick the new block underneath the active allocation block, so that + * we don't lose the use of the space remaining therein. + */ + if (set->blocks != NULL) + { + block->next = set->blocks->next; + set->blocks->next = block; + } + else + { + block->next = NULL; + set->blocks = block; + } + + set->isReset = false; + + AllocAllocInfo(set, chunk); + return AllocChunkGetPointer(chunk); + } + + /* + * Request is small enough to be treated as a chunk. Look in the + * corresponding free list to see if there is a free chunk we could reuse. + * If one is found, remove it from the free list, make it again a member + * of the alloc set and return its data address. + */ + fidx = AllocSetFreeIndex(size); + chunk = set->freelist[fidx]; + if (chunk != NULL) + { + Assert(chunk->size >= size); + + set->freelist[fidx] = (AllocChunk) chunk->aset; + + chunk->aset = (void *) set; + + #ifdef MEMORY_CONTEXT_CHECKING + chunk->requested_size = size; + /* set mark to catch clobber of "unused" space */ + if (size < chunk->size) + ((char *) AllocChunkGetPointer(chunk))[size] = 0x7E; + #endif + #ifdef RANDOMIZE_ALLOCATED_MEMORY + /* fill the allocated space with junk */ + randomize_mem((char *) AllocChunkGetPointer(chunk), size); + #endif + + /* isReset must be false already */ + Assert(!set->isReset); + + AllocAllocInfo(set, chunk); + return AllocChunkGetPointer(chunk); + } + + /* + * Choose the actual chunk size to allocate. + */ + chunk_size = (1 << ALLOC_MINBITS) << fidx; + Assert(chunk_size >= size); + + /* + * If there is enough room in the active allocation block, we will put the + * chunk into that block. Else must start a new one. + */ + if ((block = set->blocks) != NULL) + { + Size availspace = block->endptr - block->freeptr; + + if (availspace < (chunk_size + ALLOC_CHUNKHDRSZ)) + { + /* + * The existing active (top) block does not have enough room for + * the requested allocation, but it might still have a useful + * amount of space in it. Once we push it down in the block list, + * we'll never try to allocate more space from it. So, before we + * do that, carve up its free space into chunks that we can put on + * the set's freelists. + * + * Because we can only get here when there's less than + * ALLOC_CHUNK_LIMIT left in the block, this loop cannot iterate + * more than ALLOCSET_NUM_FREELISTS-1 times. + */ + while (availspace >= ((1 << ALLOC_MINBITS) + ALLOC_CHUNKHDRSZ)) + { + Size availchunk = availspace - ALLOC_CHUNKHDRSZ; + int a_fidx = AllocSetFreeIndex(availchunk); + + /* + * In most cases, we'll get back the index of the next larger + * freelist than the one we need to put this chunk on. The + * exception is when availchunk is exactly a power of 2. + */ + if (availchunk != ((Size) 1 << (a_fidx + ALLOC_MINBITS))) + { + a_fidx--; + Assert(a_fidx >= 0); + availchunk = ((Size) 1 << (a_fidx + ALLOC_MINBITS)); + } + + chunk = (AllocChunk) (block->freeptr); + + block->freeptr += (availchunk + ALLOC_CHUNKHDRSZ); + availspace -= (availchunk + ALLOC_CHUNKHDRSZ); + + chunk->size = availchunk; + #ifdef MEMORY_CONTEXT_CHECKING + chunk->requested_size = 0; /* mark it free */ + #endif + chunk->aset = (void *) set->freelist[a_fidx]; + set->freelist[a_fidx] = chunk; + } + + /* Mark that we need to create a new block */ + block = NULL; + } + } + + /* + * Time to create a new regular (multi-chunk) block? + */ + if (block == NULL) + { + Size required_size; + + /* + * The first such block has size initBlockSize, and we double the + * space in each succeeding block, but not more than maxBlockSize. + */ + blksize = set->nextBlockSize; + set->nextBlockSize <<= 1; + if (set->nextBlockSize > set->maxBlockSize) + set->nextBlockSize = set->maxBlockSize; + + /* + * If initBlockSize is less than ALLOC_CHUNK_LIMIT, we could need more + * space... but try to keep it a power of 2. + */ + required_size = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; + while (blksize < required_size) + blksize <<= 1; + + /* Try to allocate it */ + block = (AllocBlock) mmap_alloc(blksize); + + /* + * We could be asking for pretty big blocks here, so cope if malloc + * fails. But give up if there's less than a meg or so available... + */ + while (block == NULL && blksize > 1024 * 1024) + { + blksize >>= 1; + if (blksize < required_size) + break; + block = (AllocBlock) mmap_alloc(blksize); + } + + if (block == NULL) + { + MemoryContextStats(TopMemoryContext); + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) size))); + } + + block->aset = set; + block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; + block->endptr = ((char *) block) + blksize; + + /* + * If this is the first block of the set, make it the "keeper" block. + * Formerly, a keeper block could only be created during context + * creation, but allowing it to happen here lets us have fast reset + * cycling even for contexts created with minContextSize = 0; that way + * we don't have to force space to be allocated in contexts that might + * never need any space. Don't mark an oversize block as a keeper, + * however. + */ + if (set->keeper == NULL && blksize == set->initBlockSize) + set->keeper = block; + + block->next = set->blocks; + set->blocks = block; + } + + /* + * OK, do the allocation + */ + chunk = (AllocChunk) (block->freeptr); + + block->freeptr += (chunk_size + ALLOC_CHUNKHDRSZ); + Assert(block->freeptr <= block->endptr); + + chunk->aset = (void *) set; + chunk->size = chunk_size; + #ifdef MEMORY_CONTEXT_CHECKING + chunk->requested_size = size; + /* set mark to catch clobber of "unused" space */ + if (size < chunk->size) + ((char *) AllocChunkGetPointer(chunk))[size] = 0x7E; + #endif + #ifdef RANDOMIZE_ALLOCATED_MEMORY + /* fill the allocated space with junk */ + randomize_mem((char *) AllocChunkGetPointer(chunk), size); + #endif + + set->isReset = false; + + AllocAllocInfo(set, chunk); + return AllocChunkGetPointer(chunk); + } + + /* + * AllocSetFree + * Frees allocated memory; memory is removed from the set. + */ + static void + AllocSetFree(MemoryContext context, void *pointer) + { + AllocSet set = (AllocSet) context; + AllocChunk chunk = AllocPointerGetChunk(pointer); + + AllocFreeInfo(set, chunk); + + #ifdef MEMORY_CONTEXT_CHECKING + /* Test for someone scribbling on unused space in chunk */ + if (chunk->requested_size < chunk->size) + if (((char *) pointer)[chunk->requested_size] != 0x7E) + elog(WARNING, "detected write past chunk end in %s %p", + set->header.name, chunk); + #endif + + if (chunk->size > set->allocChunkLimit) + { + /* + * Big chunks are certain to have been allocated as single-chunk + * blocks. Find the containing block and return it to malloc(). + */ + AllocBlock block = set->blocks; + AllocBlock prevblock = NULL; + + while (block != NULL) + { + if (chunk == (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ)) + break; + prevblock = block; + block = block->next; + } + if (block == NULL) + elog(ERROR, "could not find block containing chunk %p", chunk); + /* let's just make sure chunk is the only one in the block */ + Assert(block->freeptr == ((char *) block) + + (chunk->size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)); + + /* OK, remove block from aset's list and free it */ + if (prevblock == NULL) + set->blocks = block->next; + else + prevblock->next = block->next; + #ifdef CLOBBER_FREED_MEMORY + /* Wipe freed memory for debugging purposes */ + memset(block, 0x7F, block->freeptr - ((char *) block)); + #endif + mmap_free(block); + } + else + { + /* Normal case, put the chunk into appropriate freelist */ + int fidx = AllocSetFreeIndex(chunk->size); + + chunk->aset = (void *) set->freelist[fidx]; + + #ifdef CLOBBER_FREED_MEMORY + /* Wipe freed memory for debugging purposes */ + memset(pointer, 0x7F, chunk->size); + #endif + + #ifdef MEMORY_CONTEXT_CHECKING + /* Reset requested_size to 0 in chunks that are on freelist */ + chunk->requested_size = 0; + #endif + set->freelist[fidx] = chunk; + } + } + + /* + * AllocSetRealloc + * Returns new pointer to allocated memory of given size; this memory + * is added to the set. Memory associated with given pointer is copied + * into the new memory, and the old memory is freed. + */ + static void * + AllocSetRealloc(MemoryContext context, void *pointer, Size size) + { + AllocSet set = (AllocSet) context; + AllocChunk chunk = AllocPointerGetChunk(pointer); + Size oldsize = chunk->size; + + #ifdef MEMORY_CONTEXT_CHECKING + /* Test for someone scribbling on unused space in chunk */ + if (chunk->requested_size < oldsize) + if (((char *) pointer)[chunk->requested_size] != 0x7E) + elog(WARNING, "detected write past chunk end in %s %p", + set->header.name, chunk); + #endif + + /* isReset must be false already */ + Assert(!set->isReset); + + /* + * Chunk sizes are aligned to power of 2 in AllocSetAlloc(). Maybe the + * allocated area already is >= the new size. (In particular, we always + * fall out here if the requested size is a decrease.) + */ + if (oldsize >= size) + { + #ifdef MEMORY_CONTEXT_CHECKING + #ifdef RANDOMIZE_ALLOCATED_MEMORY + /* We can only fill the extra space if we know the prior request */ + if (size > chunk->requested_size) + randomize_mem((char *) AllocChunkGetPointer(chunk) + chunk->requested_size, + size - chunk->requested_size); + #endif + + chunk->requested_size = size; + /* set mark to catch clobber of "unused" space */ + if (size < oldsize) + ((char *) pointer)[size] = 0x7E; + #endif + return pointer; + } + + if (oldsize > set->allocChunkLimit) + { + /* + * The chunk must have been allocated as a single-chunk block. Find + * the containing block and use realloc() to make it bigger with + * minimum space wastage. + */ + AllocBlock block = set->blocks; + AllocBlock prevblock = NULL; + Size chksize; + Size blksize; + + while (block != NULL) + { + if (chunk == (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ)) + break; + prevblock = block; + block = block->next; + } + if (block == NULL) + elog(ERROR, "could not find block containing chunk %p", chunk); + /* let's just make sure chunk is the only one in the block */ + Assert(block->freeptr == ((char *) block) + + (chunk->size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)); + + /* Do the realloc */ + chksize = MAXALIGN(size); + blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; + + block = (AllocBlock) mmap_realloc(block, blksize); + if (block == NULL) + { + MemoryContextStats(TopMemoryContext); + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) size))); + } + block->freeptr = block->endptr = ((char *) block) + blksize; + + /* Update pointers since block has likely been moved */ + chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ); + if (prevblock == NULL) + set->blocks = block; + else + prevblock->next = block; + chunk->size = chksize; + + #ifdef MEMORY_CONTEXT_CHECKING + #ifdef RANDOMIZE_ALLOCATED_MEMORY + /* We can only fill the extra space if we know the prior request */ + randomize_mem((char *) AllocChunkGetPointer(chunk) + chunk->requested_size, + size - chunk->requested_size); + #endif + + chunk->requested_size = size; + /* set mark to catch clobber of "unused" space */ + if (size < chunk->size) + ((char *) AllocChunkGetPointer(chunk))[size] = 0x7E; + #endif + + return AllocChunkGetPointer(chunk); + } + else + { + /* + * Small-chunk case. We just do this by brute force, ie, allocate a + * new chunk and copy the data. Since we know the existing data isn't + * huge, this won't involve any great memcpy expense, so it's not + * worth being smarter. (At one time we tried to avoid memcpy when it + * was possible to enlarge the chunk in-place, but that turns out to + * misbehave unpleasantly for repeated cycles of + * palloc/repalloc/pfree: the eventually freed chunks go into the + * wrong freelist for the next initial palloc request, and so we leak + * memory indefinitely. See pgsql-hackers archives for 2007-08-11.) + */ + AllocPointer newPointer; + + /* allocate new chunk */ + newPointer = AllocSetAlloc((MemoryContext) set, size); + + /* transfer existing data (certain to fit) */ + memcpy(newPointer, pointer, oldsize); + + /* free old chunk */ + AllocSetFree((MemoryContext) set, pointer); + + return newPointer; + } + } + + /* + * AllocSetGetChunkSpace + * Given a currently-allocated chunk, determine the total space + * it occupies (including all memory-allocation overhead). + */ + static Size + AllocSetGetChunkSpace(MemoryContext context, void *pointer) + { + AllocChunk chunk = AllocPointerGetChunk(pointer); + + return chunk->size + ALLOC_CHUNKHDRSZ; + } + + /* + * AllocSetIsEmpty + * Is an allocset empty of any allocated space? + */ + static bool + AllocSetIsEmpty(MemoryContext context) + { + AllocSet set = (AllocSet) context; + + /* + * For now, we say "empty" only if the context is new or just reset. We + * could examine the freelists to determine if all space has been freed, + * but it's not really worth the trouble for present uses of this + * functionality. + */ + if (set->isReset) + return true; + return false; + } + + /* + * AllocSetStats + * Displays stats about memory consumption of an allocset. + */ + static void + AllocSetStats(MemoryContext context, int level) + { + AllocSet set = (AllocSet) context; + long nblocks = 0; + long nchunks = 0; + long totalspace = 0; + long freespace = 0; + AllocBlock block; + AllocChunk chunk; + int fidx; + int i; + + for (block = set->blocks; block != NULL; block = block->next) + { + nblocks++; + totalspace += block->endptr - ((char *) block); + freespace += block->endptr - block->freeptr; + } + for (fidx = 0; fidx < ALLOCSET_NUM_FREELISTS; fidx++) + { + for (chunk = set->freelist[fidx]; chunk != NULL; + chunk = (AllocChunk) chunk->aset) + { + nchunks++; + freespace += chunk->size + ALLOC_CHUNKHDRSZ; + } + } + + for (i = 0; i < level; i++) + fprintf(stderr, " "); + + fprintf(stderr, + "%s: %lu total in %ld blocks; %lu free (%ld chunks); %lu used\n", + set->header.name, totalspace, nblocks, freespace, nchunks, + totalspace - freespace); + } + + + #ifdef MEMORY_CONTEXT_CHECKING + + /* + * AllocSetCheck + * Walk through chunks and check consistency of memory. + * + * NOTE: report errors as WARNING, *not* ERROR or FATAL. Otherwise you'll + * find yourself in an infinite loop when trouble occurs, because this + * routine will be entered again when elog cleanup tries to release memory! + */ + static void + AllocSetCheck(MemoryContext context) + { + AllocSet set = (AllocSet) context; + char *name = set->header.name; + AllocBlock block; + + for (block = set->blocks; block != NULL; block = block->next) + { + char *bpoz = ((char *) block) + ALLOC_BLOCKHDRSZ; + long blk_used = block->freeptr - bpoz; + long blk_data = 0; + long nchunks = 0; + + /* + * Empty block - empty can be keeper-block only + */ + if (!blk_used) + { + if (set->keeper != block) + elog(WARNING, "problem in alloc set %s: empty block %p", + name, block); + } + + /* + * Chunk walker + */ + while (bpoz < block->freeptr) + { + AllocChunk chunk = (AllocChunk) bpoz; + Size chsize, + dsize; + char *chdata_end; + + chsize = chunk->size; /* aligned chunk size */ + dsize = chunk->requested_size; /* real data */ + chdata_end = ((char *) chunk) + (ALLOC_CHUNKHDRSZ + dsize); + + /* + * Check chunk size + */ + if (dsize > chsize) + elog(WARNING, "problem in alloc set %s: req size > alloc size for chunk %p in block %p", + name, chunk, block); + if (chsize < (1 << ALLOC_MINBITS)) + elog(WARNING, "problem in alloc set %s: bad size %lu for chunk %p in block %p", + name, (unsigned long) chsize, chunk, block); + + /* single-chunk block? */ + if (chsize > set->allocChunkLimit && + chsize + ALLOC_CHUNKHDRSZ != blk_used) + elog(WARNING, "problem in alloc set %s: bad single-chunk %p in block %p", + name, chunk, block); + + /* + * If chunk is allocated, check for correct aset pointer. (If it's + * free, the aset is the freelist pointer, which we can't check as + * easily...) + */ + if (dsize > 0 && chunk->aset != (void *) set) + elog(WARNING, "problem in alloc set %s: bogus aset link in block %p, chunk %p", + name, block, chunk); + + /* + * Check for overwrite of "unallocated" space in chunk + */ + if (dsize > 0 && dsize < chsize && *chdata_end != 0x7E) + elog(WARNING, "problem in alloc set %s: detected write past chunk end in block %p, chunk %p", + name, block, chunk); + + blk_data += chsize; + nchunks++; + + bpoz += ALLOC_CHUNKHDRSZ + chsize; + } + + if ((blk_data + (nchunks * ALLOC_CHUNKHDRSZ)) != blk_used) + elog(WARNING, "problem in alloc set %s: found inconsistent memory block %p", + name, block); + } + } + + #endif /* MEMORY_CONTEXT_CHECKING */ *** ./contrib/dict_preload/uninstall_dict_preload.sql.orig 2010-03-18 17:00:58.000000000 +0100 --- ./contrib/dict_preload/uninstall_dict_preload.sql 2010-03-18 13:52:49.000000000 +0100 *************** *** 0 **** --- 1,10 ---- + /* $PostgreSQL: pgsql/contrib/dict_int/uninstall_dict_int.sql,v 1.3 2007/11/13 04:24:27 momjian Exp $ */ + + -- Adjust this setting to control where the objects get dropped. + SET search_path = public; + + DROP TEXT SEARCH TEMPLATE preloaddict_template CASCADE; + + DROP FUNCTION dpreloaddict_init(internal); + + DROP FUNCTION dpreloaddict_lexize(internal,internal,internal,internal); *** ./src/backend/tsearch/spell.c.orig 2010-01-02 17:57:53.000000000 +0100 --- ./src/backend/tsearch/spell.c 2010-03-31 14:41:27.412855204 +0200 *************** *** 31,36 **** --- 31,41 ---- #define tmpalloc(sz) MemoryContextAlloc(tmpCtx, (sz)) #define tmpalloc0(sz) MemoryContextAllocZero(tmpCtx, (sz)) + static void *salloc_ptr; + static Size salloc_free_size; + + #define SALLOC_BLOCKSIZE (1024 * 1024 * 2) + static void checkTmpCtx(void) { *************** *** 45,50 **** --- 50,57 ---- ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); + salloc_ptr = NULL; + salloc_free_size = 0; } else tmpCtx = CurrentMemoryContext->firstchild; *************** *** 63,68 **** --- 70,133 ---- return dst; } + /* + * used for allocation lot of small memory blocks + */ + static void * + simple_alloc(Size size) + { + void *result; + + size = MAXALIGN(size); + + if (size > salloc_free_size) + { + salloc_ptr = palloc(SALLOC_BLOCKSIZE); + salloc_free_size = SALLOC_BLOCKSIZE; + } + + result = salloc_ptr; + salloc_ptr += size; + salloc_free_size -= size; + + return result; + } + + static void * + simple_alloc0(Size size) + { + void *result; + + size = MAXALIGN(size); + + if (size > salloc_free_size) + { + salloc_ptr = palloc(SALLOC_BLOCKSIZE); + salloc_free_size = SALLOC_BLOCKSIZE; + } + + result = salloc_ptr; + salloc_ptr += size; + salloc_free_size -= size; + + memset(result, 0, size); + + return result; + } + + static char * + simple_strdup(char *str) + { + char *result; + int len; + + len = strlen(str); + result = simple_alloc(len + 1); + memcpy(result, str, len + 1); + + return result; + } + #define MAX_NORM 1024 #define MAXNORMLEN 256 *************** *** 375,383 **** Affix->flag = flag; Affix->type = type; ! Affix->find = (find && *find) ? pstrdup(find) : VoidString; if ((Affix->replen = strlen(repl)) > 0) ! Affix->repl = pstrdup(repl); else Affix->repl = VoidString; Conf->naffixes++; --- 440,448 ---- Affix->flag = flag; Affix->type = type; ! Affix->find = (find && *find) ? simple_strdup(find) : VoidString; if ((Affix->replen = strlen(repl)) > 0) ! Affix->repl = simple_strdup(repl); else Affix->repl = VoidString; Conf->naffixes++; *************** *** 833,839 **** } ptr = Conf->AffixData + Conf->nAffixData; ! *ptr = palloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) + 1 /* space */ + 1 /* \0 */ ); sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]); ptr++; --- 898,904 ---- } ptr = Conf->AffixData + Conf->nAffixData; ! *ptr = simple_alloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) + 1 /* space */ + 1 /* \0 */ ); sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]); ptr++; *************** *** 878,884 **** if (!nchar) return NULL; ! rs = (SPNode *) palloc0(SPNHDRSZ + nchar * sizeof(SPNodeData)); rs->length = nchar; data = rs->data; --- 943,949 ---- if (!nchar) return NULL; ! rs = (SPNode *) simple_alloc0(SPNHDRSZ + nchar * sizeof(SPNodeData)); rs->length = nchar; data = rs->data; *************** *** 946,952 **** int curaffix; checkTmpCtx(); - /* compress affixes */ /* Count the number of different flags used in the dictionary */ --- 1011,1016 ---- *************** *** 974,980 **** { curaffix++; Assert(curaffix < naffix); ! Conf->AffixData[curaffix] = pstrdup(Conf->Spell[i]->p.flag); } Conf->Spell[i]->p.d.affix = curaffix; --- 1038,1044 ---- { curaffix++; Assert(curaffix < naffix); ! Conf->AffixData[curaffix] = simple_strdup(Conf->Spell[i]->p.flag); } Conf->Spell[i]->p.d.affix = curaffix; *************** *** 1014,1020 **** aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1)); naff = 0; ! rs = (AffixNode *) palloc0(ANHRDSZ + nchar * sizeof(AffixNodeData)); rs->length = nchar; data = rs->data; --- 1078,1084 ---- aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1)); naff = 0; ! rs = (AffixNode *) simple_alloc0(ANHRDSZ + nchar * sizeof(AffixNodeData)); rs->length = nchar; data = rs->data; *************** *** 1030,1036 **** if (naff) { data->naff = naff; ! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } --- 1094,1100 ---- if (naff) { data->naff = naff; ! data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } *************** *** 1050,1056 **** if (naff) { data->naff = naff; ! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } --- 1114,1120 ---- if (naff) { data->naff = naff; ! data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } *************** *** 1067,1073 **** cnt = 0; int start = (issuffix) ? startsuffix : 0; int end = (issuffix) ? Conf->naffixes : startsuffix; ! AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData)); Affix->length = 1; Affix->isvoid = 1; --- 1131,1137 ---- cnt = 0; int start = (issuffix) ? startsuffix : 0; int end = (issuffix) ? Conf->naffixes : startsuffix; ! AffixNode *Affix = (AffixNode *) simple_alloc0(ANHRDSZ + sizeof(AffixNodeData)); Affix->length = 1; Affix->isvoid = 1; *************** *** 1091,1097 **** if (cnt == 0) return; ! Affix->data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * cnt); Affix->data->naff = (uint32) cnt; cnt = 0; --- 1155,1161 ---- if (cnt == 0) return; ! Affix->data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * cnt); Affix->data->naff = (uint32) cnt; cnt = 0;