diff options
Diffstat (limited to 'src/H5HFhuge.c')
| -rw-r--r-- | src/H5HFhuge.c | 1199 |
1 files changed, 1199 insertions, 0 deletions
diff --git a/src/H5HFhuge.c b/src/H5HFhuge.c new file mode 100644 index 0000000..b2a1e68 --- /dev/null +++ b/src/H5HFhuge.c @@ -0,0 +1,1199 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5HFhuge.c + * Aug 7 2006 + * Quincey Koziol <koziol@hdfgroup.org> + * + * Purpose: Routines for "huge" objects in fractal heap + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5HFmodule.h" /* This source code file is part of the H5HF module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5HFpkg.h" /* Fractal heaps */ +#include "H5MFprivate.h" /* File memory management */ +#include "H5MMprivate.h" /* Memory management */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* v2 B-tree creation macros */ +#define H5HF_HUGE_BT2_NODE_SIZE 512 +#define H5HF_HUGE_BT2_SPLIT_PERC 100 +#define H5HF_HUGE_BT2_MERGE_PERC 40 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Local v2 B-tree operations */ +static herr_t H5HF_huge_bt2_create(H5HF_hdr_t *hdr, hid_t dxpl_id); + +/* Local 'huge' object support routines */ +static hsize_t H5HF_huge_new_id(H5HF_hdr_t *hdr); +static herr_t H5HF_huge_op_real(H5HF_hdr_t *hdr, hid_t dxpl_id, + const uint8_t *id, hbool_t is_read, H5HF_operator_t op, void *op_data); + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_bt2_create + * + * Purpose: Create the v2 B-tree for tracking the huge objects in the heap + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 7 2006 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HF_huge_bt2_create(H5HF_hdr_t *hdr, hid_t dxpl_id) +{ + H5B2_create_t bt2_cparam; /* v2 B-tree creation parameters */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + + /* Compute the size of 'raw' records on disk */ + /* (Note: the size for huge IDs could be set to 'huge_id_size', instead + * of 'sizeof_size', but that would make the v2 B-tree callback routines + * depend on the heap header, which makes the v2 B-tree flush routines + * difficult to write. "Waste" an extra byte or for small heaps (where + * the 'huge_id_size' is < 'sizeof_size' in order to make this easier -QAK) + */ + if(hdr->huge_ids_direct) { + if(hdr->filter_len > 0) { + bt2_cparam.rrec_size = (size_t)((unsigned)hdr->sizeof_addr /* Address of object */ + + (unsigned)hdr->sizeof_size /* Length of object */ + + (unsigned)4 /* Filter mask for filtered object */ + + (unsigned)hdr->sizeof_size); /* Size of de-filtered object in memory */ + bt2_cparam.cls = H5HF_HUGE_BT2_FILT_DIR; + } /* end if */ + else { + bt2_cparam.rrec_size = (size_t)((unsigned)hdr->sizeof_addr /* Address of object */ + + (unsigned)hdr->sizeof_size); /* Length of object */ + bt2_cparam.cls = H5HF_HUGE_BT2_DIR; + } /* end else */ + } /* end if */ + else { + if(hdr->filter_len > 0) { + bt2_cparam.rrec_size = (size_t)((unsigned)hdr->sizeof_addr /* Address of filtered object */ + + (unsigned)hdr->sizeof_size /* Length of filtered object */ + + (unsigned)4 /* Filter mask for filtered object */ + + (unsigned)hdr->sizeof_size /* Size of de-filtered object in memory */ + + (unsigned)hdr->sizeof_size); /* Unique ID for object */ + bt2_cparam.cls = H5HF_HUGE_BT2_FILT_INDIR; + } /* end if */ + else { + bt2_cparam.rrec_size = (size_t)((unsigned)hdr->sizeof_addr /* Address of object */ + + (unsigned)hdr->sizeof_size /* Length of object */ + + (unsigned)hdr->sizeof_size); /* Unique ID for object */ + bt2_cparam.cls = H5HF_HUGE_BT2_INDIR; + } /* end else */ + } /* end else */ + bt2_cparam.node_size = (size_t)H5HF_HUGE_BT2_NODE_SIZE; + bt2_cparam.split_percent = H5HF_HUGE_BT2_SPLIT_PERC; + bt2_cparam.merge_percent = H5HF_HUGE_BT2_MERGE_PERC; + + /* Create v2 B-tree for tracking 'huge' objects */ + if(NULL == (hdr->huge_bt2 = H5B2_create(hdr->f, dxpl_id, &bt2_cparam, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTCREATE, FAIL, "can't create v2 B-tree for tracking 'huge' heap objects") + + /* Retrieve the v2 B-tree's address in the file */ + if(H5B2_get_addr(hdr->huge_bt2, &hdr->huge_bt2_addr) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't get v2 B-tree address for tracking 'huge' heap objects") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_bt2_create() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_init + * + * Purpose: Initialize information for tracking 'huge' objects + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 7 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_init(H5HF_hdr_t *hdr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* + * Check arguments. + */ + HDassert(hdr); + + /* Compute information about 'huge' objects for the heap */ + + /* Check if we can completely hold the 'huge' object's offset & length in + * the file in the heap ID (which will speed up accessing it) and we don't + * have any I/O pipeline filters. + */ +#ifdef QAK +HDfprintf(stderr, "%s: hdr->id_len = %u\n", "H5HF_huge_init", (unsigned)hdr->id_len); +HDfprintf(stderr, "%s: hdr->filter_len = %u\n", "H5HF_huge_init", (unsigned)hdr->filter_len); +#endif /* QAK */ + if(hdr->filter_len > 0) { + if((hdr->id_len - 1) >= (unsigned)(hdr->sizeof_addr + hdr->sizeof_size + 4 + hdr->sizeof_size)) { + /* Indicate that v2 B-tree doesn't have to be used to locate object */ + hdr->huge_ids_direct = TRUE; + + /* Set the size of 'huge' object IDs */ + hdr->huge_id_size = (uint8_t)(hdr->sizeof_addr + hdr->sizeof_size + hdr->sizeof_size); + } /* end if */ + else + /* Indicate that v2 B-tree must be used to access object */ + hdr->huge_ids_direct = FALSE; + } /* end if */ + else { + if((hdr->sizeof_addr + hdr->sizeof_size) <= (hdr->id_len - 1)) { + /* Indicate that v2 B-tree doesn't have to be used to locate object */ + hdr->huge_ids_direct = TRUE; + + /* Set the size of 'huge' object IDs */ + hdr->huge_id_size = (uint8_t)(hdr->sizeof_addr + hdr->sizeof_size); + } /* end if */ + else + /* Indicate that v2 B-tree must be used to locate object */ + hdr->huge_ids_direct = FALSE; + } /* end else */ + if(!hdr->huge_ids_direct) { + /* Set the size and maximum value of 'huge' object ID */ + if((hdr->id_len - 1) < sizeof(hsize_t)) { + hdr->huge_id_size = (uint8_t)(hdr->id_len - 1); + hdr->huge_max_id = ((hsize_t)1 << (hdr->huge_id_size * 8)) - 1; + } /*end if */ + else { + hdr->huge_id_size = sizeof(hsize_t); + hdr->huge_max_id = HSIZET_MAX; + } /* end else */ + } /* end if */ + hdr->huge_bt2 = NULL; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5HF_huge_init() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_new_id + * + * Purpose: Determine a new ID for an indirectly accessed 'huge' object + * (either filtered or not) + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 15 2006 + * + *------------------------------------------------------------------------- + */ +static hsize_t +H5HF_huge_new_id(H5HF_hdr_t *hdr) +{ + hsize_t new_id; /* New object's ID */ + hsize_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + + /* Check for wrapping around 'huge' object ID space */ + if(hdr->huge_ids_wrapped) + /* Fail for now - eventually should iterate through v2 B-tree, looking for available ID */ + HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, 0, "wrapping 'huge' object IDs not supported yet") + else { + /* Get new 'huge' object ID to use for object */ + /* (avoids using ID 0) */ + new_id = ++hdr->huge_next_id; + + /* Check for wrapping 'huge' object IDs around */ + if(hdr->huge_next_id == hdr->huge_max_id) + hdr->huge_ids_wrapped = TRUE; + } /* end else */ + + /* Set return value */ + ret_value = new_id; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_new_id() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_insert + * + * Purpose: Insert a 'huge' object into the file and track it + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 7 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_insert(H5HF_hdr_t *hdr, hid_t dxpl_id, size_t obj_size, void *obj, + void *_id) +{ + uint8_t *id = (uint8_t *)_id; /* Pointer to ID buffer */ + haddr_t obj_addr; /* Address of object in the file */ + void *write_buf; /* Pointer to buffer to write */ + size_t write_size; /* Size of [possibly filtered] object written to file */ + unsigned filter_mask = 0; /* Filter mask for object (only used for filtered objects) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT +#ifdef QAK +HDfprintf(stderr, "%s: obj_size = %Zu\n", FUNC, obj_size); +#endif /* QAK */ + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(obj_size > hdr->max_man_size); + HDassert(obj); + HDassert(id); + + /* Check if the v2 B-tree for tracking 'huge' heap objects has been created yet */ + if(!H5F_addr_defined(hdr->huge_bt2_addr)) { + /* Go create (& open) v2 B-tree */ + if(H5HF_huge_bt2_create(hdr, dxpl_id) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTCREATE, FAIL, "can't create v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + else { + /* Check if v2 B-tree is open yet */ + if(NULL == hdr->huge_bt2) { + /* Open existing v2 B-tree */ + if(NULL == (hdr->huge_bt2 = H5B2_open(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPENOBJ, FAIL, "unable to open v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + } /* end else */ + HDassert(hdr->huge_bt2); + + /* Check for I/O pipeline filter on heap */ + if(hdr->filter_len > 0) { + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + size_t nbytes; /* Number of bytes used */ + + /* Allocate buffer to perform I/O filtering on */ + write_size = obj_size; + if(NULL == (write_buf = H5MM_malloc(write_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + HDmemcpy(write_buf, obj, write_size); + + /* Push direct block data through I/O filter pipeline */ + nbytes = write_size; + if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_NO_EDC, + filter_cb, &nbytes, &write_size, &write_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "output pipeline failed") +#ifdef QAK +HDfprintf(stderr, "%s: nbytes = %Zu, write_size = %Zu, write_buf = %p\n", FUNC, nbytes, write_size, write_buf); +HDfprintf(stderr, "%s: obj_size = %Zu, obj = %p\n", FUNC, obj_size, obj); +#endif /* QAK */ + + /* Update size of object on disk */ + write_size = nbytes; + } /* end if */ + else { + write_buf = obj; + write_size = obj_size; + } /* end else */ + + /* Allocate space in the file for storing the 'huge' object */ + if(HADDR_UNDEF == (obj_addr = H5MF_alloc(hdr->f, H5FD_MEM_FHEAP_HUGE_OBJ, dxpl_id, (hsize_t)write_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap huge object") + + /* Write the object's data to disk */ + if(H5F_block_write(hdr->f, H5FD_MEM_FHEAP_HUGE_OBJ, obj_addr, write_size, H5AC_rawdata_dxpl_id, write_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_WRITEERROR, FAIL, "writing 'huge' object to file failed") + + /* Release buffer for writing, if we had one */ + if(write_buf != obj) { + HDassert(hdr->filter_len > 0); + H5MM_xfree(write_buf); + } /* end if */ + + /* Perform different actions for directly & indirectly accessed 'huge' objects */ + if(hdr->huge_ids_direct) { + if(hdr->filter_len > 0) { + H5HF_huge_bt2_filt_dir_rec_t obj_rec; /* Record for tracking object */ + + /* Initialize record for tracking object in v2 B-tree */ + obj_rec.addr = obj_addr; + obj_rec.len = write_size; + obj_rec.filter_mask = filter_mask; + obj_rec.obj_size = obj_size; +#ifdef QAK +HDfprintf(stderr, "%s: obj_rec = {%a, %Hu, %x, %Hu}\n", FUNC, obj_rec.addr, obj_rec.len, obj_rec.filter_mask, obj_rec.obj_size); +#endif /* QAK */ + + /* Insert record for object in v2 B-tree */ + if(H5B2_insert(hdr->huge_bt2, dxpl_id, &obj_rec) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINSERT, FAIL, "couldn't insert object tracking record in v2 B-tree") + + /* Encode ID for user */ + *id++ = H5HF_ID_VERS_CURR | H5HF_ID_TYPE_HUGE; + H5F_addr_encode(hdr->f, &id, obj_addr); + H5F_ENCODE_LENGTH(hdr->f, id, (hsize_t)write_size); + UINT32ENCODE(id, filter_mask); + H5F_ENCODE_LENGTH(hdr->f, id, (hsize_t)obj_size); + } /* end if */ + else { + H5HF_huge_bt2_dir_rec_t obj_rec; /* Record for tracking object */ + + /* Initialize record for tracking object in v2 B-tree */ + obj_rec.addr = obj_addr; + obj_rec.len = write_size; +#ifdef QAK +HDfprintf(stderr, "%s: obj_rec = {%a, %Hu}\n", FUNC, obj_rec.addr, obj_rec.len); +#endif /* QAK */ + + /* Insert record for object in v2 B-tree */ + if(H5B2_insert(hdr->huge_bt2, dxpl_id, &obj_rec) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINSERT, FAIL, "couldn't insert object tracking record in v2 B-tree") + + /* Encode ID for user */ + *id++ = H5HF_ID_VERS_CURR | H5HF_ID_TYPE_HUGE; + H5F_addr_encode(hdr->f, &id, obj_addr); + H5F_ENCODE_LENGTH(hdr->f, id, (hsize_t)write_size); + } /* end if */ + } /* end if */ + else { + H5HF_huge_bt2_filt_indir_rec_t filt_indir_rec; /* Record for tracking filtered object */ + H5HF_huge_bt2_indir_rec_t indir_rec; /* Record for tracking non-filtered object */ + void *ins_rec; /* Pointer to record to insert */ + hsize_t new_id; /* New ID for object */ + + /* Get new ID for object */ + if(0 == (new_id = H5HF_huge_new_id(hdr))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't generate new ID for object") + + if(hdr->filter_len > 0) { + /* Initialize record for object in v2 B-tree */ + filt_indir_rec.addr = obj_addr; + filt_indir_rec.len = write_size; + filt_indir_rec.filter_mask = filter_mask; + filt_indir_rec.obj_size = obj_size; + filt_indir_rec.id = new_id; +#ifdef QAK +HDfprintf(stderr, "%s: filt_indir_rec = {%a, %Hu, %x, %Hu, %Hu}\n", FUNC, filt_indir_rec.addr, filt_indir_rec.len, filt_indir_rec.filter_mask, filt_indir_rec.obj_size, filt_indir_rec.id); +#endif /* QAK */ + + /* Set pointer to record to insert */ + ins_rec = &filt_indir_rec; + } /* end if */ + else { + /* Initialize record for object in v2 B-tree */ + indir_rec.addr = obj_addr; + indir_rec.len = write_size; + indir_rec.id = new_id; +#ifdef QAK +HDfprintf(stderr, "%s: indir_rec = {%a, %Hu, %Hu}\n", FUNC, indir_rec.addr, indir_rec.len, indir_rec.id); +#endif /* QAK */ + + /* Set pointer to record to insert */ + ins_rec = &indir_rec; + } /* end else */ + + /* Insert record for tracking object in v2 B-tree */ + if(H5B2_insert(hdr->huge_bt2, dxpl_id, ins_rec) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINSERT, FAIL, "couldn't insert object tracking record in v2 B-tree") + + /* Encode ID for user */ + *id++ = H5HF_ID_VERS_CURR | H5HF_ID_TYPE_HUGE; + UINT64ENCODE_VAR(id, new_id, hdr->huge_id_size) + } /* end else */ + + /* Update statistics about heap */ + hdr->huge_size += obj_size; + hdr->huge_nobjs++; + + /* Mark heap header as modified */ + if(H5HF_hdr_dirty(hdr) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDIRTY, FAIL, "can't mark heap header as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_insert() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_get_obj_len + * + * Purpose: Get the size of a 'huge' object in a fractal heap + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 8 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_get_obj_len(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, + size_t *obj_len_p) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + HDassert(id); + HDassert(obj_len_p); + + /* Skip over the flag byte */ + id++; + + /* Check if 'huge' object ID encodes address & length directly */ + if(hdr->huge_ids_direct) { + if(hdr->filter_len > 0) { + /* Skip over filtered object info */ + id += hdr->sizeof_addr + hdr->sizeof_size + 4; + + /* Retrieve the object's length */ + H5F_DECODE_LENGTH(hdr->f, id, *obj_len_p); + } /* end if */ + else { + /* Skip over object offset in file */ + id += hdr->sizeof_addr; + + /* Retrieve the object's length */ + H5F_DECODE_LENGTH(hdr->f, id, *obj_len_p); + } /* end else */ + } /* end if */ + else { + /* Check if v2 B-tree is open yet */ + if(NULL == hdr->huge_bt2) { + /* Open existing v2 B-tree */ + if(NULL == (hdr->huge_bt2 = H5B2_open(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPENOBJ, FAIL, "unable to open v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + + if(hdr->filter_len > 0) { + H5HF_huge_bt2_filt_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_filt_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_filt_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's length */ + *obj_len_p = (size_t)found_rec.obj_size; + } /* end if */ + else { + H5HF_huge_bt2_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's length */ + *obj_len_p = (size_t)found_rec.len; + } /* end else */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_get_obj_len() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__huge_get_obj_off + * + * Purpose: Get the offset of a 'huge' object in a fractal heap + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 8 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF__huge_get_obj_off(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, + hsize_t *obj_off_p) +{ + haddr_t obj_addr; /* Object's address in the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + HDassert(id); + HDassert(obj_off_p); + + /* Skip over the flag byte */ + id++; + + /* Check if 'huge' object ID encodes address & length directly */ + if(hdr->huge_ids_direct) { + /* Retrieve the object's address (common) */ + H5F_addr_decode(hdr->f, &id, &obj_addr); + } /* end if */ + else { + /* Sanity check */ + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + + /* Check if v2 B-tree is open yet */ + if(NULL == hdr->huge_bt2) { + /* Open existing v2 B-tree */ + if(NULL == (hdr->huge_bt2 = H5B2_open(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPENOBJ, FAIL, "unable to open v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + + if(hdr->filter_len > 0) { + H5HF_huge_bt2_filt_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_filt_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_filt_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's address & length */ + obj_addr = found_rec.addr; + } /* end if */ + else { + H5HF_huge_bt2_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's address & length */ + obj_addr = found_rec.addr; + } /* end else */ + } /* end else */ + + /* Set the value to return */ + *obj_off_p = (hsize_t)obj_addr; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__huge_get_obj_off() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_op_real + * + * Purpose: Internal routine to perform an operation on a 'huge' object + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 8 2006 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HF_huge_op_real(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, + hbool_t is_read, H5HF_operator_t op, void *op_data) +{ + void *read_buf = NULL; /* Pointer to buffer for reading */ + haddr_t obj_addr; /* Object's address in the file */ + size_t obj_size = 0; /* Object's size in the file */ + unsigned filter_mask = 0; /* Filter mask for object (only used for filtered objects) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(id); + HDassert(is_read || op); + + /* Skip over the flag byte */ + id++; + + /* Check for 'huge' object ID that encodes address & length directly */ + if(hdr->huge_ids_direct) { + /* Retrieve the object's address and length (common) */ + H5F_addr_decode(hdr->f, &id, &obj_addr); + H5F_DECODE_LENGTH(hdr->f, id, obj_size); + + /* Retrieve extra information needed for filtered objects */ + if(hdr->filter_len > 0) + UINT32DECODE(id, filter_mask); + } /* end if */ + else { + /* Sanity check */ + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + + /* Check if v2 B-tree is open yet */ + if(NULL == hdr->huge_bt2) { + /* Open existing v2 B-tree */ + if(NULL == (hdr->huge_bt2 = H5B2_open(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPENOBJ, FAIL, "unable to open v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + + if(hdr->filter_len > 0) { + H5HF_huge_bt2_filt_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_filt_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_filt_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's address & length */ + obj_addr = found_rec.addr; + H5_CHECKED_ASSIGN(obj_size, size_t, found_rec.len, hsize_t); + filter_mask = found_rec.filter_mask; + } /* end if */ + else { + H5HF_huge_bt2_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's address & length */ + obj_addr = found_rec.addr; + H5_CHECKED_ASSIGN(obj_size, size_t, found_rec.len, hsize_t); + } /* end else */ + } /* end else */ + + /* Set up buffer for reading */ + if(hdr->filter_len > 0 || !is_read) { + if(NULL == (read_buf = H5MM_malloc((size_t)obj_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + } /* end if */ + else + read_buf = op_data; + + /* Read the object's (possibly filtered) data from the file */ + /* (reads directly into application's buffer if no filters are present) */ + if(H5F_block_read(hdr->f, H5FD_MEM_FHEAP_HUGE_OBJ, obj_addr, (size_t)obj_size, H5AC_rawdata_dxpl_id, read_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_READERROR, FAIL, "can't read 'huge' object's data from the file") + + /* Check for I/O pipeline filter on heap */ + if(hdr->filter_len > 0) { + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + size_t read_size; /* Object's size in the file */ + size_t nbytes; /* Number of bytes used */ + + /* De-filter the object */ + read_size = nbytes = obj_size; + if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_NO_EDC, filter_cb, &nbytes, &read_size, &read_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "input filter failed") + obj_size = nbytes; + } /* end if */ + + /* Perform correct operation on buffer read in */ + if(is_read) { + /* Copy object to user's buffer if there's filters on heap data */ + /* (if there's no filters, the object was read directly into the user's buffer) */ + if(hdr->filter_len > 0) + HDmemcpy(op_data, read_buf, (size_t)obj_size); + } /* end if */ + else { + /* Call the user's 'op' callback */ + if(op(read_buf, (size_t)obj_size, op_data) < 0) { + /* Release buffer */ + read_buf = H5MM_xfree(read_buf); + + /* Indicate error */ + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPERATE, FAIL, "application's callback failed") + } /* end if */ + } /* end if */ + +done: + /* Release the buffer for reading */ + if(read_buf && read_buf != op_data) + read_buf = H5MM_xfree(read_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_op_real() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_write + * + * Purpose: Write a 'huge' object to the heap + * + * Note: This implementation somewhat limited: it doesn't handle + * heaps with filters, which would require re-compressing the + * huge object and probably changing the address of the object + * on disk (and possibly the heap ID for "direct" huge IDs). + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Feb 21 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_write(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, + const void *obj) +{ + haddr_t obj_addr; /* Object's address in the file */ + size_t obj_size; /* Object's size in the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(id); + HDassert(obj); + + /* Check for filters on the heap */ + if(hdr->filter_len > 0) + HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, FAIL, "modifying 'huge' object with filters not supported yet") + + /* Skip over the flag byte */ + id++; + + /* Check for 'huge' object ID that encodes address & length directly */ + if(hdr->huge_ids_direct) { + /* Retrieve the object's address and length (common) */ + H5F_addr_decode(hdr->f, &id, &obj_addr); + H5F_DECODE_LENGTH(hdr->f, id, obj_size); + } /* end if */ + else { + H5HF_huge_bt2_indir_rec_t found_rec; /* Record found from tracking object */ + H5HF_huge_bt2_indir_rec_t search_rec; /* Record for searching for object */ + + /* Sanity check */ + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + + /* Check if v2 B-tree is open yet */ + if(NULL == hdr->huge_bt2) { + /* Open existing v2 B-tree */ + if(NULL == (hdr->huge_bt2 = H5B2_open(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPENOBJ, FAIL, "unable to open v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Look up object in v2 B-tree */ + if(H5B2_find(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_indir_found, &found_rec) != TRUE) + HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in B-tree") + + /* Retrieve the object's address & length */ + obj_addr = found_rec.addr; + H5_CHECKED_ASSIGN(obj_size, size_t, found_rec.len, hsize_t); + } /* end else */ + + /* Write the object's data to the file */ + /* (writes directly from application's buffer) */ + if(H5F_block_write(hdr->f, H5FD_MEM_FHEAP_HUGE_OBJ, obj_addr, obj_size, H5AC_rawdata_dxpl_id, obj) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_WRITEERROR, FAIL, "writing 'huge' object to file failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_read + * + * Purpose: Read a 'huge' object from the heap + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Sept 11 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_read(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, void *obj) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(id); + HDassert(obj); + + /* Call the internal 'op' routine */ + if(H5HF_huge_op_real(hdr, dxpl_id, id, TRUE, NULL, obj) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPERATE, FAIL, "unable to operate on heap object") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_op + * + * Purpose: Operate directly on a 'huge' object + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Sept 11 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_op(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, + H5HF_operator_t op, void *op_data) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(id); + HDassert(op); + + /* Call the internal 'op' routine routine */ + if(H5HF_huge_op_real(hdr, dxpl_id, id, FALSE, op, op_data) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPERATE, FAIL, "unable to operate on heap object") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_op() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_remove + * + * Purpose: Remove a 'huge' object from the file and the v2 B-tree tracker + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 8 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_remove(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id) +{ + H5HF_huge_remove_ud_t udata; /* User callback data for v2 B-tree remove call */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + HDassert(id); + + /* Check if v2 B-tree is open yet */ + if(NULL == hdr->huge_bt2) { + /* Open existing v2 B-tree */ + if(NULL == (hdr->huge_bt2 = H5B2_open(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTOPENOBJ, FAIL, "unable to open v2 B-tree for tracking 'huge' heap objects") + } /* end if */ + + /* Skip over the flag byte */ + id++; + + /* Set up the common callback info */ + udata.hdr = hdr; + udata.dxpl_id = dxpl_id; + + /* Check for 'huge' object ID that encodes address & length directly */ + if(hdr->huge_ids_direct) { + if(hdr->filter_len > 0) { + H5HF_huge_bt2_filt_dir_rec_t search_rec; /* Record for searching for object */ + + /* Retrieve the object's address and length */ + /* (used as key in v2 B-tree record) */ + H5F_addr_decode(hdr->f, &id, &search_rec.addr); + H5F_DECODE_LENGTH(hdr->f, id, search_rec.len); + + /* Remove the record for tracking the 'huge' object from the v2 B-tree */ + /* (space in the file for the object is freed in the 'remove' callback) */ + if(H5B2_remove(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_filt_dir_remove, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTREMOVE, FAIL, "can't remove object from B-tree") + } /* end if */ + else { + H5HF_huge_bt2_dir_rec_t search_rec; /* Record for searching for object */ + + /* Retrieve the object's address and length */ + /* (used as key in v2 B-tree record) */ + H5F_addr_decode(hdr->f, &id, &search_rec.addr); + H5F_DECODE_LENGTH(hdr->f, id, search_rec.len); + + /* Remove the record for tracking the 'huge' object from the v2 B-tree */ + /* (space in the file for the object is freed in the 'remove' callback) */ + if(H5B2_remove(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_dir_remove, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTREMOVE, FAIL, "can't remove object from B-tree") + } /* end else */ + } /* end if */ + else { + if(hdr->filter_len > 0) { + H5HF_huge_bt2_filt_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Remove the record for tracking the 'huge' object from the v2 B-tree */ + /* (space in the file for the object is freed in the 'remove' callback) */ + if(H5B2_remove(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_filt_indir_remove, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTREMOVE, FAIL, "can't remove object from B-tree") + } /* end if */ + else { + H5HF_huge_bt2_indir_rec_t search_rec; /* Record for searching for object */ + + /* Get ID for looking up 'huge' object in v2 B-tree */ + UINT64DECODE_VAR(id, search_rec.id, hdr->huge_id_size) + + /* Remove the record for tracking the 'huge' object from the v2 B-tree */ + /* (space in the file for the object is freed in the 'remove' callback) */ + if(H5B2_remove(hdr->huge_bt2, dxpl_id, &search_rec, H5HF__huge_bt2_indir_remove, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTREMOVE, FAIL, "can't remove object from B-tree") + } /* end else */ + } /* end else */ + + /* Update statistics about heap */ + hdr->huge_size -= udata.obj_len; + hdr->huge_nobjs--; + + /* Mark heap header as modified */ + if(H5HF_hdr_dirty(hdr) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDIRTY, FAIL, "can't mark heap header as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_remove() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_term + * + * Purpose: Shut down the information for tracking 'huge' objects + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 8 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_term(H5HF_hdr_t *hdr, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + + /* Check if v2 B-tree index is open */ + if(hdr->huge_bt2) { + /* Sanity check */ + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + + /* Close v2 B-tree index */ + if(H5B2_close(hdr->huge_bt2, dxpl_id) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTCLOSEOBJ, FAIL, "can't close v2 B-tree") + hdr->huge_bt2 = NULL; + } /* end if */ + + /* Check if there are no more 'huge' objects in the heap and delete the + * v2 B-tree that tracks them, if so + */ + if(H5F_addr_defined(hdr->huge_bt2_addr) && hdr->huge_nobjs == 0) { + /* Sanity check */ + HDassert(hdr->huge_size == 0); + + /* Delete the v2 B-tree */ + /* (any v2 B-tree class will work here) */ + if(H5B2_delete(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f, NULL, NULL) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDELETE, FAIL, "can't delete v2 B-tree") + + /* Reset the information about 'huge' objects in the file */ + hdr->huge_bt2_addr = HADDR_UNDEF; + hdr->huge_next_id = 0; + hdr->huge_ids_wrapped = FALSE; + + /* Mark heap header as modified */ + if(H5HF_hdr_dirty(hdr) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDIRTY, FAIL, "can't mark heap header as dirty") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_term() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF_huge_delete + * + * Purpose: Delete all the 'huge' objects in the heap, and the v2 B-tree + * tracker for them + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 8 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_huge_delete(H5HF_hdr_t *hdr, hid_t dxpl_id) +{ + H5HF_huge_remove_ud_t udata; /* User callback data for v2 B-tree remove call */ + H5B2_remove_t op; /* Callback for v2 B-tree removal */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(hdr); + HDassert(H5F_addr_defined(hdr->huge_bt2_addr)); + HDassert(hdr->huge_nobjs); + HDassert(hdr->huge_size); + + /* Set up the callback info */ + udata.hdr = hdr; + udata.dxpl_id = dxpl_id; + + /* Set the v2 B-tree callback operator */ + if(hdr->huge_ids_direct) { + if(hdr->filter_len > 0) + op = H5HF__huge_bt2_filt_dir_remove; + else + op = H5HF__huge_bt2_dir_remove; + } /* end if */ + else { + if(hdr->filter_len > 0) + op = H5HF__huge_bt2_filt_indir_remove; + else + op = H5HF__huge_bt2_indir_remove; + } /* end else */ + + /* Delete the v2 B-tree */ + if(H5B2_delete(hdr->f, dxpl_id, hdr->huge_bt2_addr, hdr->f, op, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDELETE, FAIL, "can't delete v2 B-tree") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_huge_delete() */ + |