/*- * See the file LICENSE for redistribution information. * * Copyright (c) 1999, 2000 * Loic Dachary. All rights reserved. * * TODO: * Keith Bostic says: * The only change I'd probably think about is if * we should merge the call to CDB___memp_pg and CDB___memp_cmpr -- kind * of a stack of page modification routines, that sits on top of * CDB___os_io. That's a bigger change, but it's probably cleaner * in the long-run. * * Pending questions: * * The CMPR structure contains binary data. Should we store them in network order ? * How is this related to DB_AM_SWAP ? * * The calls to cmpr_open/cmpr_close in memp_pgread/pgwrite are probably not * at the right place / most logical place in the function. I have troubles * finding out where to put them. They work, that's not the problem. Just don't * know if they should be before or after. * * When opening weakcmpr DB, the DB_THREAD flag is not available. Should it be * set if the main DB has this flag set ? * * In CDB___memp_cmpr, niop is always multiplied by compression factor for page 0. * I see no problems with this but it's a bit awkward. * * In CDB___memp_cmpr_page, the page built fills some fields of the PAGE structure * others are set to 0. I'm not 100% sure this is enough. It should only impact * utilities that read pages by incrementing pgno. Only stat does this an it's * enough for it. I've not found any other context where these fake pages are * used. * */ #include "db_config.h" #ifndef lint static const char sccsid[] = "@(#)mp_cmpr.c 1.1 (Senga) 01/08/99"; #endif /* not lint */ #ifndef NO_SYSTEM_INCLUDES #include #include #endif #include "db_int.h" #include "db_page.h" #include "shqueue.h" #include "db_shash.h" #include "mp.h" #include "db_page.h" #include "common_ext.h" #ifdef DEBUG #include "WordMonitor.h" #endif /* DEBUG */ #if 0 #define DEBUG_CMPR 1 #endif #if 0 #define DEBUG_CMPR_ALLOC 1 #endif /* * Helpers declarations. */ static int CDB___memp_cmpr_page __P((DB_MPOOLFILE *, CMPR *, DB_IO *, ssize_t *)); /* * Maximum chain length */ #define CMPR_MAX (dbenv->mp_cmpr_info->max_npages) #define CMPR_MULTIPLY(n) ((n) << (dbenv->mp_cmpr_info->coefficient)) #define CMPR_DIVIDE(n) ((n) >> (dbenv->mp_cmpr_info->coefficient)) #ifdef HAVE_LIBZ static DB_CMPR_INFO default_cmpr_info = { CDB___memp_cmpr_deflate, CDB___memp_cmpr_inflate, 1, 3, NULL }; #else /* HAVE_LIBZ */ static DB_CMPR_INFO default_cmpr_info = { 0, 0, 0, 0, NULL }; #endif /* HAVE_LIBZ */ /* * Entry point. Functionaly equivalent to CDB___os_io. * Compress/uncompress pages before returning them or writing them to disk. */ /* * CDB___memp_cmpr -- * Transparent compression read/write * * PUBLIC: int CDB___memp_cmpr __P((DB_MPOOLFILE *, BH *, DB_IO *, int, ssize_t *)); */ int CDB___memp_cmpr(dbmfp, bhp, db_io, flag, niop) DB_MPOOLFILE *dbmfp; BH *bhp; DB_IO *db_io; int flag; ssize_t *niop; { size_t orig_pagesize = db_io->pagesize; db_pgno_t orig_pgno = db_io->pgno; size_t orig_bytes = db_io->bytes; DB_ENV *dbenv = dbmfp->dbmp->dbenv; int ret = 0; db_io->pagesize = CMPR_DIVIDE(db_io->pagesize); db_io->bytes = CMPR_DIVIDE(db_io->bytes); /* * Page 0 is a special case. It contains the metadata information (at most 512 bytes) * and must not be compressed because it is read with CDB___os_read and not CDB___os_io. */ switch (flag) { case DB_IO_READ: if(db_io->pgno == 0) { ret = CDB___os_io(db_io, DB_IO_READ, niop); *niop = CMPR_MULTIPLY(*niop); } else ret = CDB___memp_cmpr_read(dbmfp, bhp, db_io, niop); break; case DB_IO_WRITE: if(db_io->pgno == 0) { ret = CDB___os_io(db_io, DB_IO_WRITE, niop); *niop = CMPR_MULTIPLY(*niop); } else ret = CDB___memp_cmpr_write(dbmfp, bhp, db_io, niop); break; } db_io->pgno = orig_pgno; db_io->pagesize = orig_pagesize; db_io->bytes = orig_bytes; return ret; } /* * CDB___memp_cmpr_read -- * Transparent compression read * * PUBLIC: int CDB___memp_cmpr_read __P((DB_MPOOLFILE *, BH *, DB_IO *, ssize_t *)); */ int CDB___memp_cmpr_read(dbmfp, bhp, db_io, niop) DB_MPOOLFILE *dbmfp; BH *bhp; DB_IO *db_io; ssize_t *niop; { CMPR cmpr; int ret; int chain = 0; u_int8_t *buffcmpr = 0; int buffcmpr_length = 0; int chain_length = 0; db_pgno_t first_pgno = db_io->pgno; DB_ENV *dbenv = dbmfp->dbmp->dbenv; DB_CMPR_INFO *cmpr_info = dbenv->mp_cmpr_info; /* * By default the compression does not use too much space, * hence the chain is empty. */ F_CLR(bhp, BH_CMPR); /* * Read first page (if no overflow, this is the only one) */ ret = CDB___os_io(db_io, DB_IO_READ, niop); /* * An error or partial read on the first page means that we're not * going anywhere. */ if(ret || *niop < db_io->pagesize) goto err; /* * Read the cmpr header from page. */ memcpy(&cmpr, db_io->buf, sizeof(CMPR)); /* * If not at the beginning of compressed page chain, build * a fake page. */ if(F_ISSET(&cmpr, DB_CMPR_FREE) || F_ISSET(&cmpr, DB_CMPR_INTERNAL)) { ret = CDB___memp_cmpr_page(dbmfp, &cmpr, db_io, niop); goto err; } /* * Sanity check. Happens if file corrupted. */ if(!F_ISSET(&cmpr, DB_CMPR_FIRST)) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: expected DB_CMPR_FIRST flag set at pgno = %ld", db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL); goto err; } if((ret = CDB___os_malloc(db_io->pagesize * CMPR_MAX, NULL, &buffcmpr)) != 0) goto err; do { /* * Read the first part of the compressed data from page. */ memcpy(buffcmpr + buffcmpr_length, DB_CMPR_DATA(db_io), DB_CMPR_PAGESIZE(db_io)); buffcmpr_length += DB_CMPR_PAGESIZE(db_io); /* * Flag must only contain FIRST|INTERNAL and/or CHAIN. If other bits are * set, the data is corrupted. Removing the FIRST|INTERNAL bits and checking * the CHAIN bit with == instead of F_ISSET verify this. */ F_CLR(&cmpr, DB_CMPR_FIRST | DB_CMPR_INTERNAL); chain = cmpr.flags; if(chain == DB_CMPR_CHAIN) { /* * Overflow Case. Continue reading data from extra pages. */ chain_length++; if(chain_length >= CMPR_MAX) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: compression chain too long at pgno = %ld", db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL); goto err; } if(cmpr.next == 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: cmpr.next is null at pgno = %ld", chain, db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL); goto err; } /* * Keep the chain in buffer header. */ CDB___memp_cmpr_alloc_chain(dbmfp->dbmp, bhp, BH_CMPR_POOL); bhp->chain[chain_length - 1] = cmpr.next; db_io->pgno = cmpr.next; /* * Read data from extra page. */ if((ret = CDB___os_io(db_io, DB_IO_READ, niop)) != 0 || *niop != db_io->pagesize) { ret = EIO; goto err; } /* * Read the cmpr header from this extra page */ memcpy(&cmpr, db_io->buf, sizeof(CMPR)); } else if(chain != 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: unexpected compression flag value 0x%x at pgno = %ld", chain, db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); goto err; } else if(cmpr.next != 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: cmpr.next is not null at pgno = %ld", chain, db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); goto err; } } while(chain); /* * We gathered all the compressed data in buffcmpr, inflate it. */ if((ret = (*cmpr_info->uncompress)(buffcmpr, buffcmpr_length, db_io->buf, CMPR_MULTIPLY(db_io->pagesize), cmpr_info->user_data)) != 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: unable to uncompress page at pgno = %ld", first_pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); goto err; } #ifdef DEBUG { int ratio = buffcmpr_length > 0 ? (CMPR_MULTIPLY(db_io->pagesize) / buffcmpr_length) : 0; if(ratio > 10) ratio = 10; word_monitor_add(WORD_MONITOR_COMPRESS_01 + ratio, 1); } #endif /* DEBUG */ *niop = CMPR_MULTIPLY(db_io->pagesize); err: #ifdef DEBUG_CMPR if(chain_length > 0) { int i; fprintf(stderr,"CDB___memp_cmpr_read:: chain_length (number of overflow pages):%2d\n",chain_length); fprintf(stderr,"CDB___memp_cmpr_read:: chain "); for(i = 0; i < chain_length; i++) fprintf(stderr, "%d, ", bhp->chain[i]); fprintf(stderr, "\n"); } #endif if(buffcmpr) CDB___os_free(buffcmpr, 0); return ret; } /* * CDB___memp_cmpr_write -- * Transparent compression write * * PUBLIC: int CDB___memp_cmpr_write __P((DB_MPOOLFILE *, BH *, DB_IO *, ssize_t *)); */ int CDB___memp_cmpr_write(dbmfp, bhp, db_io, niop) DB_MPOOLFILE *dbmfp; BH *bhp; DB_IO *db_io; ssize_t *niop; { CMPR cmpr; int chain_length = 0; int first_nonreused_chain_pos = 0; int ret; u_int8_t *buffcmpr = 0; u_int8_t *buffp; int buffcmpr_length; u_int8_t *orig_buff = db_io->buf; DB_ENV *dbenv = dbmfp->dbmp->dbenv; DB_CMPR_INFO *cmpr_info = dbenv->mp_cmpr_info; if((ret = CDB___os_malloc(CMPR_MULTIPLY(db_io->bytes), NULL, &db_io->buf)) != 0) goto err; if((ret = (*cmpr_info->compress)(orig_buff, CMPR_MULTIPLY(db_io->pagesize), &buffcmpr, &buffcmpr_length, cmpr_info->user_data)) != 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: unable to compress page at pgno = %ld", db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); goto err; } #ifdef DEBUG { int ratio = buffcmpr_length > 0 ? (CMPR_MULTIPLY(db_io->pagesize) / buffcmpr_length) : 0; if(ratio > 10) ratio = 10; word_monitor_add(WORD_MONITOR_COMPRESS_01 + ratio, 1); } #endif /* DEBUG */ /* * This can never happen. */ if(buffcmpr_length > DB_CMPR_PAGESIZE(db_io) * CMPR_MAX) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: compressed data is too big at pgno = %ld", db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL); goto err; } buffp = buffcmpr; cmpr.flags = DB_CMPR_FIRST; cmpr.next = 0; /* write pages until the whole compressed data is written */ do { int length = buffcmpr_length - (buffp - buffcmpr); int copy_length = length > DB_CMPR_PAGESIZE(db_io) ? DB_CMPR_PAGESIZE(db_io) : length; /* * We handle serious compression stuff only if we need to. * overflow! the compressed buffer is too big -> get extra page */ if(length > copy_length) { chain_length++; if(chain_length >= CMPR_MAX) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: chain_length overflow"); ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL); goto err; } F_SET(&cmpr, DB_CMPR_CHAIN); if((ret = CDB___memp_cmpr_alloc(dbmfp, &cmpr.next, bhp, &first_nonreused_chain_pos)) != 0) goto err; CDB___memp_cmpr_alloc_chain(dbmfp->dbmp, bhp, BH_CMPR_OS); bhp->chain[chain_length - 1] = cmpr.next; } /* write in the cmpr header */ memcpy(db_io->buf, &cmpr, DB_CMPR_OVERHEAD); /* write in what's left of the compressed buffer (and that also fits in) */ memcpy(db_io->buf + DB_CMPR_OVERHEAD, buffp, copy_length); buffp += copy_length; /* actual output */ if((ret = CDB___os_io(db_io, DB_IO_WRITE, niop)) != 0 || *niop != db_io->pagesize) { ret = EIO; goto err; } db_io->pgno = cmpr.next; cmpr.flags = DB_CMPR_INTERNAL; cmpr.next = 0; } while(buffp - buffcmpr < buffcmpr_length); #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_write:: chain_length (number of overflow pages):%2d\n",chain_length); if(chain_length > 0) { int i; fprintf(stderr,"CDB___memp_cmpr_write:: chain "); for(i = 0; i < chain_length; i++) fprintf(stderr, "%d, ", bhp->chain[i]); fprintf(stderr, "\n"); } #endif /* * If the chain was not completely reused, free the remaining pages (the page compression * rate is better). */ if(F_ISSET(bhp, BH_CMPR) && first_nonreused_chain_pos >= 0) { int i; CMPR cmpr; cmpr.flags = DB_CMPR_FREE; cmpr.next = 0; memcpy(db_io->buf, &cmpr, sizeof(CMPR)); for(i = first_nonreused_chain_pos; i < (CMPR_MAX - 1) && bhp->chain[i]; i++) { if((ret = CDB___memp_cmpr_free(dbmfp, bhp->chain[i])) != 0) goto err; /* * Mark the page as free for recovery. */ db_io->pgno = bhp->chain[i]; if((ret = CDB___os_io(db_io, DB_IO_WRITE, niop)) != 0 || *niop != db_io->pagesize) { ret = EIO; goto err; } bhp->chain[i] = 0; } } CDB___memp_cmpr_free_chain(dbmfp->dbmp, bhp); /* * In case of success, always pretend that we exactly wrote the * all bytes of the original pagesize. */ *niop = CMPR_MULTIPLY(db_io->pagesize); err: CDB___os_free(db_io->buf, 0); db_io->buf = orig_buff; if(buffcmpr) CDB___os_free(buffcmpr, 0); return ret; } /* * Helpers */ /* * CDB___memp_cmpr_page -- * Build a fake page. This function is a CDB___memp_cmpr_read helper. * */ static int CDB___memp_cmpr_page(dbmfp, cmpr, db_io, niop) DB_MPOOLFILE *dbmfp; CMPR *cmpr; DB_IO *db_io; ssize_t *niop; { DB_ENV *dbenv = dbmfp->dbmp->dbenv; int ret = 0; PAGE page; memset((char*)&page, '\0', sizeof(PAGE)); page.pgno = db_io->pgno; page.type = F_ISSET(cmpr, DB_CMPR_FREE) ? P_CMPR_FREE : P_CMPR_INTERNAL; /* * Sanity check */ if(db_io->pagesize < sizeof(PAGE)) { ret = ENOMEM; goto err; } memcpy(db_io->buf, (char*)&page, sizeof(PAGE)); *niop = CMPR_MULTIPLY(db_io->pagesize); err: return ret; } #ifdef HAVE_LIBZ #include "zlib.h" #endif /* HAVE_LIBZ */ /* * CDB___memp_cmpr_inflate -- * Decompress buffer * * PUBLIC: int CDB___memp_cmpr_inflate __P((const u_int8_t *, int, u_int8_t *, int, void *)); */ int CDB___memp_cmpr_inflate(inbuff, inbuff_length, outbuff, outbuff_length, user_data) const u_int8_t* inbuff; int inbuff_length; u_int8_t* outbuff; int outbuff_length; void *user_data; { #ifdef HAVE_LIBZ int ret = 0; z_stream c_stream; c_stream.zalloc=(alloc_func)0; c_stream.zfree=(free_func)0; c_stream.opaque=(voidpf)0; c_stream.next_in = (Bytef*)inbuff; c_stream.avail_in = inbuff_length; c_stream.next_out = outbuff; c_stream.avail_out = outbuff_length; if(inflateInit(&c_stream) != Z_OK || inflate(&c_stream, Z_FINISH) != Z_STREAM_END || inflateEnd(&c_stream) != Z_OK) ret = EIO; /* * The uncompressed data must *exactly* fill outbuff_length. */ if(c_stream.avail_out != 0) ret = EIO; return ret; #else /* HAVE_LIBZ */ return EINVAL; #endif /* HAVE_LIBZ */ } /* * CDB___memp_cmpr_deflate -- * Compress buffer * * PUBLIC: int CDB___memp_cmpr_deflate __P((const u_int8_t *, int, u_int8_t **, int*, void *)); */ int CDB___memp_cmpr_deflate(inbuff, inbuff_length, outbuffp, outbuff_lengthp, user_data) const u_int8_t* inbuff; int inbuff_length; u_int8_t** outbuffp; int* outbuff_lengthp; void *user_data; { #ifdef HAVE_LIBZ int ret = 0; int r; z_stream c_stream; u_int8_t* outbuff; /* * Z_FINISH can be used immediately after deflateInit if all the compression * is to be done in a single step. In this case, avail_out must be at least * 0.1% larger than avail_in plus 12 bytes. If deflate does not return * Z_STREAM_END, then it must be called again as described above. * * !!! * In order to avoid division by 1000, divide by 512 (2^9) using shift. * That is, make the buffer 0.2% larger. */ int outbuff_length = inbuff_length + (inbuff_length >> 9) + 12; *outbuffp = 0; *outbuff_lengthp = 0; if(CDB___os_malloc(outbuff_length, NULL, &outbuff) != 0) { ret = ENOMEM; goto err; } /* * Clear possible garbage in the page */ { PAGE* pg = (PAGE*)inbuff; switch(TYPE(pg)) { case P_IBTREE: case P_LBTREE: memset((char*)(inbuff + LOFFSET(pg)), '\0', P_FREESPACE(pg)); break; } } c_stream.zalloc=(alloc_func)0; c_stream.zfree=(free_func)0; c_stream.opaque=(voidpf)0; if(deflateInit(&c_stream, Z_DEFAULT_COMPRESSION) != Z_OK) { ret = EIO; goto err; } c_stream.next_in = (Bytef*)inbuff; c_stream.avail_in = inbuff_length; c_stream.next_out = outbuff; c_stream.avail_out = outbuff_length; while((r = deflate(&c_stream, Z_FINISH)) != Z_STREAM_END && r == Z_OK) ; if(r != Z_STREAM_END) ret = EIO; if(deflateEnd(&c_stream) != Z_OK) ret = EIO; if(ret == 0) { *outbuffp = outbuff; *outbuff_lengthp = outbuff_length - c_stream.avail_out; } else { CDB___os_free(outbuff, outbuff_length); } #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_deflate:: compress %d bytes to %d \n", inbuff_length, *outbuff_lengthp); #endif err: return ret; #else /* HAVE_LIBZ */ return EINVAL; #endif /* HAVE_LIBZ */ } /* * CDB___memp_cmpr_info_valid -- * Compute compressed page size */ static int CDB___memp_cmpr_info_valid(dbenv,cmpr_info) DB_ENV *dbenv; DB_CMPR_INFO *cmpr_info; { int ret = 0; if(!cmpr_info ) { CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: cmpr_info == NULL"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } if(!cmpr_info->compress ) { CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: compress == NULL!"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } if(!cmpr_info->uncompress ) { CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: uncompress == NULL!"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } if(cmpr_info->coefficient == 0 || cmpr_info->coefficient > 5 ) { CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: coefficient should be > 0 and < 5 coefficient=%d ", cmpr_info->coefficient); ret = CDB___db_panic(dbenv, EINVAL); goto err; } if(cmpr_info->max_npages == 0 || cmpr_info->max_npages > 128 ) { CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: max_npages should be > 0 and < 128 max_npages=%d ", cmpr_info->max_npages); ret = CDB___db_panic(dbenv, EINVAL); goto err; } err: return ret; } /* * __memp_cmpr_pagesize -- * Compute compressed page size * * PUBLIC: u_int8_t CDB___memp_cmpr_coefficient __P((DB_ENV *dbenv)); */ u_int8_t CDB___memp_cmpr_coefficient(dbenv) DB_ENV *dbenv; { u_int8_t ret = 0; if(!dbenv || !dbenv->mp_cmpr_info) { ret = default_cmpr_info.coefficient; } else { CDB___memp_cmpr_info_valid(dbenv, dbenv->mp_cmpr_info); ret = dbenv->mp_cmpr_info->coefficient; } return (ret); } /* * Initialisation of page compression */ /* * CDB___memp_cmpr_open -- * Open the db that contains the free compression pages. * * PUBLIC: int CDB___memp_cmpr_open __P((const char *, DB_ENV *, CMPR_CONTEXT *)); */ int CDB___memp_cmpr_open(dbenv, path, flags, mode, cmpr_context) DB_ENV *dbenv; const char *path; int flags; int mode; CMPR_CONTEXT *cmpr_context; { int ret; char* tmp = 0; int tmp_length = strlen(path) + strlen(DB_CMPR_SUFFIX) + 1; /* * Management of pages containing data when the compression does not achieve * the expected compression ratio. */ { DB *dbp; if((ret = CDB___os_malloc(tmp_length, NULL, &tmp)) != 0) goto err; sprintf(tmp, "%s%s", path, DB_CMPR_SUFFIX); if(CDB_db_create(&dbp, dbenv, 0) != 0) goto err; cmpr_context->weakcmpr = dbp; (dbp->set_flags)(dbp, DB_RECNUM); LF_CLR(DB_COMPRESS); if(!LF_ISSET(DB_RDONLY)) LF_SET(DB_CREATE); if((ret = (dbp->open)(dbp, tmp, NULL, DB_BTREE, flags, mode)) != 0) goto err; } /* * Initialisation of cmpr_context */ if(!dbenv->mp_cmpr_info) { if(default_cmpr_info.compress == 0) { CDB___db_err(dbenv, "CDB___memp_cmpr_open: zlib compression not available, re-compile --with-zlib=DIR"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } dbenv->mp_cmpr_info = &default_cmpr_info; } /* * Check if cmpr_info is sane */ if((ret = CDB___memp_cmpr_info_valid(dbenv, dbenv->mp_cmpr_info))) goto err; err: if(tmp) CDB___os_free(tmp, tmp_length); return ret; } /* * CDB___memp_cmpr_close -- * Close the db that contains the free compression pages. * * PUBLIC: int CDB___memp_cmpr_close __P((CMPR_CONTEXT *)); */ int CDB___memp_cmpr_close(cmpr_context) CMPR_CONTEXT *cmpr_context; { int ret = 0; if(cmpr_context->weakcmpr == 0) { ret = EINVAL; goto err; } if((ret = cmpr_context->weakcmpr->close(cmpr_context->weakcmpr, 0)) != 0) goto err; cmpr_context->weakcmpr = 0; err: return ret; } /* * CDB___memp_cmpr_alloc -- * Get a new free page to store weak compression data. * * PUBLIC: int CDB___memp_cmpr_alloc __P((DB_MPOOLFILE *, db_pgno_t *, BH *, int *)); */ int CDB___memp_cmpr_alloc(dbmfp, pgnop, bhp, first_nonreused_chain_posp) DB_MPOOLFILE *dbmfp; db_pgno_t *pgnop; BH *bhp; int *first_nonreused_chain_posp; { DB_ENV *dbenv = dbmfp->dbmp->dbenv; int ret = 0; #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_alloc:: bhp:%8x bhp->chain:%8x first_nonreused_chain_posp:%2d\n", bhp, bhp->chain, *first_nonreused_chain_posp); #endif if(F_ISSET(bhp, BH_CMPR) && bhp->chain == NULL) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: BH_CMPR set and bhp->chain == NULL"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } if((*first_nonreused_chain_posp) >= (CMPR_MAX - 1)) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: first_nonreused_chain_pos >= (CMPR_MAX - 1)"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } /* * If possible reuse an existing chain. */ if((*first_nonreused_chain_posp) >= 0 && F_ISSET(bhp, BH_CMPR) && bhp->chain[*first_nonreused_chain_posp]) { *pgnop = bhp->chain[*first_nonreused_chain_posp]; (*first_nonreused_chain_posp)++; #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_alloc:: reusing page in chain \n"); #endif } else { DB *db = dbmfp->cmpr_context.weakcmpr; DBT key; DBT data; db_recno_t recno = 1; /* all pages in bhp->chain are now reused */ (*first_nonreused_chain_posp) = -1; #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_alloc:: no more reusable pages in chain\n"); #endif if(db == 0) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: dbmfp->cmpr_context.weakcmpr is null"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } /* * If the existing chain is too short, pop a free page from * the free pages database. */ memset(&key, '\0', sizeof(DBT)); memset(&data, '\0', sizeof(DBT)); key.data = &recno; key.size = sizeof(recno); if((ret = db->get(db, NULL, &key, &data, DB_SET_RECNO)) != 0) { /* * If the free list is empty, create a new page. */ #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_alloc:: weakcmpr free page pool empty, allocating\n"); #endif if(ret == DB_NOTFOUND) { DB_MPOOL *dbmp = dbmfp->dbmp; ret = 0; R_LOCK(dbenv, &dbmp->reginfo); ++dbmfp->mfp->last_pgno; #ifdef DEBUG word_monitor_set(WORD_MONITOR_PGNO, dbmfp->mfp->last_pgno); #endif /* DEBUG */ *pgnop = dbmfp->mfp->last_pgno; R_UNLOCK(dbenv, &dbmp->reginfo); ret = 0; } else { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: unexpected error from weakcmpr base"); ret = CDB___db_panic(dbenv, ret); } } else { if(key.size != sizeof(db_pgno_t)) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: unexpected key size from weakcmpr base (%d instead of %d)", key.size, sizeof(db_pgno_t)); ret = CDB___db_panic(dbenv, ret); } else { memcpy((char*)pgnop, (char*)key.data, key.size); if((ret = db->del(db, NULL, &key, 0)) != 0) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: del error, got pgno %d", *pgnop); ret = CDB___db_panic(dbenv, ret); } if(*pgnop == 0) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: unexpected pgno == 0"); ret = CDB___db_panic(dbenv, ret); } } #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_alloc:: reuse free page %d from weakcmpr\n", *pgnop); #endif } } err: return ret; } /* * CDB___memp_cmpr_free -- * Release a page used to store weak compression data. * * PUBLIC: int CDB___memp_cmpr_free __P((DB_MPOOLFILE *, db_pgno_t)); */ int CDB___memp_cmpr_free(dbmfp, pgno) DB_MPOOLFILE *dbmfp; db_pgno_t pgno; { int ret = 0; DB_ENV *dbenv = dbmfp->dbmp->dbenv; DB *db = dbmfp->cmpr_context.weakcmpr; DBT key; DBT data; #ifdef DEBUG_CMPR fprintf(stderr,"CDB___memp_cmpr_free:: freeing page (inserting into weakcmpr):%3d \n",pgno); #endif if(db == 0) { CDB___db_err(dbenv, "CDB___memp_cmpr_free: dbmfp->cmpr_context.weakcmpr is null"); ret = CDB___db_panic(dbenv, EINVAL); goto err; } memset(&key, '\0', sizeof(DBT)); memset(&data, '\0', sizeof(DBT)); key.data = &pgno; key.size = sizeof(db_pgno_t); data.data = " "; data.size = 1; if((ret = db->put(db, 0, &key, &data, DB_NOOVERWRITE)) != 0) { CDB___db_err(dbenv, "CDB___memp_cmpr_free: put failed for pgno = %d", pgno); ret = CDB___db_panic(dbenv, ret); goto err; } err: return ret; } /* * CDB___memp_cmpr_alloc_chain -- * Allocate chain entry in BH * * PUBLIC: int CDB___memp_cmpr_alloc_chain __P((DB_MPOOL *, BH *)); */ int CDB___memp_cmpr_alloc_chain(dbmp, bhp, alloc_type) DB_MPOOL *dbmp; BH *bhp; int alloc_type; { DB_ENV *dbenv = dbmp->dbenv; int ret = 0; if(!bhp->chain) { int alloc_ret; int alloc_length = sizeof(db_pgno_t)*(CMPR_MAX-1); switch(alloc_type) { case BH_CMPR_POOL: { MPOOL *mp = dbmp->reginfo.primary; int n_cache = NCACHE(mp, bhp->pgno); alloc_ret = CDB___memp_alloc(dbmp, &dbmp->c_reginfo[n_cache], NULL, alloc_length, NULL, (void *)(&bhp->chain)); F_SET(bhp, BH_CMPR_POOL); } break; case BH_CMPR_OS: alloc_ret = CDB___os_malloc(alloc_length, NULL, &bhp->chain); F_SET(bhp, BH_CMPR_OS); break; default: CDB___db_err(dbenv, "CDB___memp_cmpr_alloc_chain: unknown alloc type :%d", alloc_type); ret = CDB___db_panic(dbenv, EINVAL); goto err; break; } if(alloc_ret) { CDB___db_err(dbenv, "CDB___memp_cmpr_alloc_chain: memp_alloc %d bytes failed:%d", alloc_length, alloc_ret); ret = CDB___db_panic(dbenv, EINVAL); goto err; } memset((void *)bhp->chain, 0, alloc_length); #if defined(DEBUG_CMPR) || defined(DEBUG_CMPR_ALLOC) fprintf(stderr, "CDB___memp_cmpr_alloc_chain:: allocate chain in %s :%8x\n", (alloc_type == BH_CMPR_OS ? "malloc" : "shalloc"), bhp->chain); #endif } else { #ifdef DEBUG_CMPR fprintf(stderr, "CDB___memp_cmpr_alloc_chain:: existing chain:%8x\n", bhp->chain); #endif } F_SET(bhp, BH_CMPR); err: return ret; } /* * CDB___memp_cmpr_free_chain -- * Free chain entry in BH * * PUBLIC: int CDB___memp_cmpr_free_chain __P((DB_MPOOL *, BH *)); */ int CDB___memp_cmpr_free_chain(dbmp, bhp) DB_MPOOL *dbmp; BH *bhp; { DB_ENV *dbenv = dbmp->dbenv; if(F_ISSET(bhp, BH_CMPR)) { if(bhp->chain) { int alloc_length = sizeof(db_pgno_t)*(CMPR_MAX-1); int alloc_type = bhp->flags & (BH_CMPR_POOL|BH_CMPR_OS); switch(alloc_type) { case BH_CMPR_POOL: { MPOOL *mp = dbmp->reginfo.primary; int n_cache = NCACHE(mp, bhp->pgno); CDB___db_shalloc_free(dbmp->c_reginfo[n_cache].addr, bhp->chain); } break; case BH_CMPR_OS: CDB___os_free(bhp->chain, alloc_length); break; default: CDB___db_err(dbenv, "CDB___memp_cmpr_free_chain: unknown alloc type :%d", alloc_type); return CDB___db_panic(dbenv, EINVAL); break; } #if defined(DEBUG_CMPR) || defined(DEBUG_CMPR_ALLOC) fprintf(stderr, "CDB___memp_cmpr_free_chain:: free chain in %s :%8x\n", (alloc_type == BH_CMPR_OS ? "malloc" : "shalloc"), bhp->chain); #endif bhp->chain = NULL; } else { CDB___db_err(dbenv, "CDB___memp_cmpr_free_chain: BH_CMPR set but null bhp->chain"); return CDB___db_panic(dbenv, EINVAL); } } else if(bhp->chain) { CDB___db_err(dbenv, "CDB___memp_cmpr_free_chain: BH_CMPR not set but bhp->chain not null"); return CDB___db_panic(dbenv, EINVAL); } F_CLR(bhp, BH_CMPR | BH_CMPR_OS | BH_CMPR_POOL); return 0; }