From 30dab1578bf97cd722789683aa3132ba47ce17b9 Mon Sep 17 00:00:00 2001 From: Suben Saha Date: Wed, 6 Aug 2025 17:37:50 -0700 Subject: [PATCH 1/3] wip --- src/api/pdc_client_connect.c | 77 +- src/commons/collections/include/pdc_dllist.h | 1235 +++++++++++++++++ src/commons/collections/include/pdc_pht.h | 96 ++ src/commons/collections/include/pdc_set.h | 11 + src/commons/collections/pdc_dllist.c | 73 + src/commons/collections/pdc_pht.c | 73 + src/commons/collections/pdc_set.c | 24 + src/server/include/pdc_client_server_common.h | 37 + src/server/include/pdc_server_metadata.h | 10 + src/server/include/pdc_server_metadata_pht.h | 0 src/server/pdc_client_server_common.c | 73 + src/server/pdc_server.c | 2 + src/server/pdc_server_metadata.c | 80 ++ src/server/pdc_server_metadata_pht.c | 162 +++ 14 files changed, 1952 insertions(+), 1 deletion(-) create mode 100644 src/commons/collections/include/pdc_dllist.h create mode 100644 src/commons/collections/include/pdc_pht.h create mode 100644 src/commons/collections/pdc_dllist.c create mode 100644 src/commons/collections/pdc_pht.c create mode 100644 src/server/include/pdc_server_metadata_pht.h create mode 100644 src/server/pdc_server_metadata_pht.c diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index ca2cb7ff2..81115ba6b 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -66,7 +66,11 @@ #include #include #include -#include +#include "pdc_timing.h" +#include "errno.h" + +/* #define TANG_DEBUG 1 */ +#define RING_ARC_SIZE 64 int is_client_debug_g = 0; pdc_server_selection_t pdc_server_selection_g = PDC_SERVER_DEFAULT; @@ -144,6 +148,7 @@ static hg_id_t metadata_delete_by_id_register_id_g; static hg_id_t metadata_update_register_id_g; static hg_id_t metadata_add_tag_register_id_g; static hg_id_t metadata_add_kvtag_register_id_g; +static hg_id_t metadata_check_prefix_register_id_g; static hg_id_t metadata_del_kvtag_register_id_g; static hg_id_t metadata_get_kvtag_register_id_g; static hg_id_t region_lock_register_id_g; @@ -1341,6 +1346,7 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po metadata_update_register_id_g = PDC_metadata_update_register(*hg_class); metadata_add_tag_register_id_g = PDC_metadata_add_tag_register(*hg_class); metadata_add_kvtag_register_id_g = PDC_metadata_add_kvtag_register(*hg_class); + metadata_check_prefix_register_id_g = PDC_metadata_check_prefix_register(*hg_class); metadata_del_kvtag_register_id_g = PDC_metadata_del_kvtag_register(*hg_class); metadata_get_kvtag_register_id_g = PDC_metadata_get_kvtag_register(*hg_class); region_lock_register_id_g = PDC_region_lock_register(*hg_class); @@ -6394,6 +6400,75 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) FUNC_LEAVE(ret_value); } +/*** + * Check if the prefix exists in the PHT + * @param server: The server to check against + * @param prefix: The prefix to check + * + * This function will send mercury RPC request to the server and check if the prefix exists + */ + +static perr_t +PDC_check_prefix(uint32_t server, char *prefix){ + +} + +uint32_t +PDC_get_server_using_pht(uint32_t key_hash) { + int ring_size = pdc_server_num_g * RING_ARC_SIZE; + uint64_t mapped_hash = key_hash % ring_size; + + for (int i = 0; i < pdc_server_num_g; i++) { + int node_pos = (i + 1) * ceil(ring_size / (float)pdc_server_num_g); + + if (mapped_hash <= node_pos) { + return i; + } + } + + // Wrap around + return 0; +} + +uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix, bool is_leaf){ + int target_server = -1; + int max = strlen(prefix) - 1; + int mid = 1; + char *slice; + + while (mid <= max){ + slice = malloc((mid + 1) * sizeof(char)); + strncpy(slice, prefix, mid); + slice[mid] = '\0'; + char *prefix_bin = string_to_binary(slice); + uint32_t hash_value = prefix_hash(prefix_bin); + target_server = PDC_get_server_using_pht(hash_value); + //void *resp = PDC_check_if_prefix_exists(target_server, slice); + + /** TODO: Check if the prefix bucket is redirected, make another call here */ + + *target_prefix = malloc((mid + 1) * sizeof(char)); + strncpy(*target_prefix, slice, mid); + (*target_prefix)[mid] = '\0'; + + /*** If the prefix exists and a leaf node, return the target server */ + + + free(slice); + return target_server; + } + +} + +static perr_t +PDC_add_metadata_key(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { + char *prefix; + char *key_name; + perr_t ret_value = SUCCEED; + + key_name = kvtag->name; + prefix = string_to_binary(key_name); +} static hg_return_t metadata_get_kvtag_rpc_cb(const struct hg_cb_info *callback_info) diff --git a/src/commons/collections/include/pdc_dllist.h b/src/commons/collections/include/pdc_dllist.h new file mode 100644 index 000000000..c6cef2774 --- /dev/null +++ b/src/commons/collections/include/pdc_dllist.h @@ -0,0 +1,1235 @@ +/* +Copyright (c) 2007-2017, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTLIST_H +#define UTLIST_H + +#define UTLIST_VERSION 2.0.2 + +#include +#include +#include "pdc_set.h" +/* + * This file contains macros to manipulate singly and doubly-linked lists. + * + * 1. LL_ macros: singly-linked lists. + * 2. DL_ macros: doubly-linked lists. + * 3. CDL_ macros: circular doubly-linked lists. + * + * To use singly-linked lists, your structure must have a "next" pointer. + * To use doubly-linked lists, your structure must "prev" and "next" pointers. + * Either way, the pointer to the head of the list must be initialized to NULL. + * + * ----------------.EXAMPLE ------------------------- + * struct item { + * int id; + * struct item *prev, *next; + * } + * + * struct item *list = NULL: + * + * int main() { + * struct item *item; + * ... allocate and populate item ... + * DL_APPEND(list, item); + * } + **/ +typedef void* DoublyLinkedListKey; +typedef void* DoublyLinkedListValue; +typedef struct _DoublyLinkedList DoublyLinkedList; +typedef struct _DoublyLinkedListItem DoublyLinkedListItem; + +/** + * Hash function used to generate hash values for keys used in a hash + * table. + * + * @param data The data to generate a hash value for. + * @return The hash value. + */ + +typedef unsigned int (*DoublyLinkedListHashFunc)(DoublyLinkedListKey data); + +/** + * Function used to compare two keys for equality. + * + * @return Non-zero if the two keys are equal, zero if the keys are + * not equal. + */ + +typedef int (*DoublyLinkedListEqualFunc)(DoublyLinkedListKey value1, DoublyLinkedListKey value2); + +/** + * Type of function used to free keys when entries are removed from a + * hash table. + */ + +typedef void (*DoublyLinkedListKeyFreeFunc)(DoublyLinkedListKey value); + +DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func, DoublyLinkedListKeyFreeFunc key_free_func); +int dllist_insert(DoublyLinkedList *list, void *key, void *value); +int *dllist_search_key(DoublyLinkedList *list, void *key); +int *dllist_search_range(DoublyLinkedList *list, void *start_key, bool include_start, void *end_key, bool include_end); +int dllist_destroy(void *list); + /** + * -------------------------------------------------- + * + * For doubly-linked lists, the append and delete macros are O(1) + * For singly-linked lists, append and delete are O(n) but prepend is O(1) + * The sort macro is O(n log(n)) for all types of single/double/circular lists. + */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ source) this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#if !defined(LDECLTYPE) && !defined(NO_DECLTYPE) +#if defined(_MSC_VER) /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define LDECLTYPE(x) decltype(x) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#endif +#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__) +#define NO_DECLTYPE +#else /* GNU, Sun and other compilers */ +#define LDECLTYPE(x) __typeof(x) +#endif +#endif + +/* for VS2008 we use some workarounds to get around the lack of decltype, + * namely, we always reassign our tmp variable to the list head if we need + * to dereference its prev/next pointers, and save/restore the real head.*/ +#ifdef NO_DECLTYPE +#define IF_NO_DECLTYPE(x) x +#define LDECLTYPE(x) char * +#define UTLIST_SV(elt, list) \ + _tmp = (char *)(list); \ + { \ + char **_alias = (char **)&(list); \ + *_alias = (elt); \ + } +#define UTLIST_NEXT(elt, list, next) ((char *)((list)->next)) +#define UTLIST_NEXTASGN(elt, list, to, next) \ + { \ + char **_alias = (char **)&((list)->next); \ + *_alias = (char *)(to); \ + } +#define UTLIST_PREVASGN(elt, list, to, prev) \ + { \ + char **_alias = (char **)&((list)->prev); \ + *_alias = (char *)(to); \ + } +#define UTLIST_RS(list) \ + { \ + char **_alias = (char **)&(list); \ + *_alias = _tmp; \ + } +#define UTLIST_CASTASGN(a, b) \ + { \ + char **_alias = (char **)&(a); \ + *_alias = (char *)(b); \ + } +#else +#define IF_NO_DECLTYPE(x) +#define UTLIST_SV(elt, list) +#define UTLIST_NEXT(elt, list, next) ((elt)->next) +#define UTLIST_NEXTASGN(elt, list, to, next) ((elt)->next) = (to) +#define UTLIST_PREVASGN(elt, list, to, prev) ((elt)->prev) = (to) +#define UTLIST_RS(list) +#define UTLIST_CASTASGN(a, b) (a) = (b) +#endif + +/****************************************************************************** + * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort * + * Unwieldy variable names used here to avoid shadowing passed-in variables. * + *****************************************************************************/ +#define LL_SORT(list, cmp) LL_SORT2(list, cmp, next) + +#define LL_SORT2(list, cmp, next) \ + do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + UTLIST_CASTASGN(_ls_p, list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + if (!_ls_q) \ + break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } \ + else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } \ + else if (cmp(_ls_p, _ls_q) <= 0) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } \ + else { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _ls_e, next); \ + UTLIST_RS(list); \ + } \ + else { \ + UTLIST_CASTASGN(list, _ls_e); \ + } \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, NULL, next); \ + UTLIST_RS(list); \ + } \ + if (_ls_nmerges <= 1) { \ + _ls_looping = 0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ + } while (0) + +#define DL_SORT(list, cmp) DL_SORT2(list, cmp, prev, next) + +#define DL_SORT2(list, cmp, prev, next) \ + do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + UTLIST_CASTASGN(_ls_p, list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + if (!_ls_q) \ + break; \ + } \ + _ls_qsize = _ls_insize; \ + while ((_ls_psize > 0) || ((_ls_qsize > 0) && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } \ + else if ((_ls_qsize == 0) || (!_ls_q)) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } \ + else if (cmp(_ls_p, _ls_q) <= 0) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + } \ + else { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _ls_e, next); \ + UTLIST_RS(list); \ + } \ + else { \ + UTLIST_CASTASGN(list, _ls_e); \ + } \ + UTLIST_SV(_ls_e, list); \ + UTLIST_PREVASGN(_ls_e, list, _ls_tail, prev); \ + UTLIST_RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + UTLIST_CASTASGN((list)->prev, _ls_tail); \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, NULL, next); \ + UTLIST_RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping = 0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ + } while (0) + +#define CDL_SORT(list, cmp) CDL_SORT2(list, cmp, prev, next) + +#define CDL_SORT2(list, cmp, prev, next) \ + do { \ + LDECLTYPE(list) _ls_p; \ + LDECLTYPE(list) _ls_q; \ + LDECLTYPE(list) _ls_e; \ + LDECLTYPE(list) _ls_tail; \ + LDECLTYPE(list) _ls_oldhead; \ + LDECLTYPE(list) _tmp; \ + int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ + if (list) { \ + _ls_insize = 1; \ + _ls_looping = 1; \ + while (_ls_looping) { \ + UTLIST_CASTASGN(_ls_p, list); \ + UTLIST_CASTASGN(_ls_oldhead, list); \ + (list) = NULL; \ + _ls_tail = NULL; \ + _ls_nmerges = 0; \ + while (_ls_p) { \ + _ls_nmerges++; \ + _ls_q = _ls_p; \ + _ls_psize = 0; \ + for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ + _ls_psize++; \ + UTLIST_SV(_ls_q, list); \ + if (UTLIST_NEXT(_ls_q, list, next) == _ls_oldhead) { \ + _ls_q = NULL; \ + } \ + else { \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + } \ + UTLIST_RS(list); \ + if (!_ls_q) \ + break; \ + } \ + _ls_qsize = _ls_insize; \ + while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ + if (_ls_psize == 0) { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { \ + _ls_q = NULL; \ + } \ + } \ + else if (_ls_qsize == 0 || !_ls_q) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + if (_ls_p == _ls_oldhead) { \ + _ls_p = NULL; \ + } \ + } \ + else if (cmp(_ls_p, _ls_q) <= 0) { \ + _ls_e = _ls_p; \ + UTLIST_SV(_ls_p, list); \ + _ls_p = UTLIST_NEXT(_ls_p, list, next); \ + UTLIST_RS(list); \ + _ls_psize--; \ + if (_ls_p == _ls_oldhead) { \ + _ls_p = NULL; \ + } \ + } \ + else { \ + _ls_e = _ls_q; \ + UTLIST_SV(_ls_q, list); \ + _ls_q = UTLIST_NEXT(_ls_q, list, next); \ + UTLIST_RS(list); \ + _ls_qsize--; \ + if (_ls_q == _ls_oldhead) { \ + _ls_q = NULL; \ + } \ + } \ + if (_ls_tail) { \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _ls_e, next); \ + UTLIST_RS(list); \ + } \ + else { \ + UTLIST_CASTASGN(list, _ls_e); \ + } \ + UTLIST_SV(_ls_e, list); \ + UTLIST_PREVASGN(_ls_e, list, _ls_tail, prev); \ + UTLIST_RS(list); \ + _ls_tail = _ls_e; \ + } \ + _ls_p = _ls_q; \ + } \ + UTLIST_CASTASGN((list)->prev, _ls_tail); \ + UTLIST_CASTASGN(_tmp, list); \ + UTLIST_SV(_ls_tail, list); \ + UTLIST_NEXTASGN(_ls_tail, list, _tmp, next); \ + UTLIST_RS(list); \ + if (_ls_nmerges <= 1) { \ + _ls_looping = 0; \ + } \ + _ls_insize *= 2; \ + } \ + } \ + } while (0) + +/****************************************************************************** + * singly linked list macros (non-circular) * + *****************************************************************************/ +#define LL_PREPEND(head, add) LL_PREPEND2(head, add, next) + +#define LL_PREPEND2(head, add, next) \ + do { \ + (add)->next = (head); \ + (head) = (add); \ + } while (0) + +#define LL_CONCAT(head1, head2) LL_CONCAT2(head1, head2, next) + +#define LL_CONCAT2(head1, head2, next) \ + do { \ + LDECLTYPE(head1) _tmp; \ + if (head1) { \ + _tmp = (head1); \ + while (_tmp->next) { \ + _tmp = _tmp->next; \ + } \ + _tmp->next = (head2); \ + } \ + else { \ + (head1) = (head2); \ + } \ + } while (0) + +#define LL_APPEND(head, add) LL_APPEND2(head, add, next) + +#define LL_APPEND2(head, add, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + (add)->next = NULL; \ + if (head) { \ + _tmp = (head); \ + while (_tmp->next) { \ + _tmp = _tmp->next; \ + } \ + _tmp->next = (add); \ + } \ + else { \ + (head) = (add); \ + } \ + } while (0) + +#define LL_INSERT_INORDER(head, add, cmp) LL_INSERT_INORDER2(head, add, cmp, next) + +#define LL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if (head) { \ + LL_LOWER_BOUND(head, _tmp, add, cmp); \ + LL_APPEND_ELEM(head, _tmp, add); \ + } \ + else { \ + (head) = (add); \ + (head)->next = NULL; \ + } \ + } while (0) + +#define LL_LOWER_BOUND(head, elt, like, cmp) LL_LOWER_BOUND2(head, elt, like, cmp, next) + +#define LL_LOWER_BOUND2(head, elt, like, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, like)) >= 0) { \ + (elt) = NULL; \ + } \ + else { \ + for ((elt) = (head); (elt)->next != NULL; (elt) = (elt)->next) { \ + if (cmp((elt)->next, like) >= 0) { \ + break; \ + } \ + } \ + } \ + } while (0) + +#define LL_DELETE(head, del) LL_DELETE2(head, del, next) + +#define LL_DELETE2(head, del, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if ((head) == (del)) { \ + (head) = (head)->next; \ + } \ + else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (del))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (del)->next; \ + } \ + } \ + } while (0) + +#define LL_COUNT(head, el, counter) LL_COUNT2(head, el, counter, next) + +#define LL_COUNT2(head, el, counter, next) \ + do { \ + (counter) = 0; \ + LL_FOREACH2(head, el, next) \ + { \ + ++(counter); \ + } \ + } while (0) + +#define LL_FOREACH(head, el) LL_FOREACH2(head, el, next) + +#define LL_FOREACH2(head, el, next) for ((el) = (head); el; (el) = (el)->next) + +#define LL_FOREACH_SAFE(head, el, tmp) LL_FOREACH_SAFE2(head, el, tmp, next) + +#define LL_FOREACH_SAFE2(head, el, tmp, next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +#define LL_SEARCH_SCALAR(head, out, field, val) LL_SEARCH_SCALAR2(head, out, field, val, next) + +#define LL_SEARCH_SCALAR2(head, out, field, val, next) \ + do { \ + LL_FOREACH2(head, out, next) \ + { \ + if ((out)->field == (val)) \ + break; \ + } \ + } while (0) + +#define LL_SEARCH(head, out, elt, cmp) LL_SEARCH2(head, out, elt, cmp, next) + +#define LL_SEARCH2(head, out, elt, cmp, next) \ + do { \ + LL_FOREACH2(head, out, next) \ + { \ + if ((cmp(out, elt)) == 0) \ + break; \ + } \ + } while (0) + +#define LL_REPLACE_ELEM2(head, el, add, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ + } while (0) + +#define LL_REPLACE_ELEM(head, el, add) LL_REPLACE_ELEM2(head, el, add, next) + +#define LL_PREPEND_ELEM2(head, el, add, next) \ + do { \ + if (el) { \ + LDECLTYPE(head) _tmp; \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + else { \ + _tmp = (head); \ + while (_tmp->next && (_tmp->next != (el))) { \ + _tmp = _tmp->next; \ + } \ + if (_tmp->next) { \ + _tmp->next = (add); \ + } \ + } \ + } \ + else { \ + LL_APPEND2(head, add, next); \ + } \ + } while (0) + +#define LL_PREPEND_ELEM(head, el, add) LL_PREPEND_ELEM2(head, el, add, next) + +#define LL_APPEND_ELEM2(head, el, add, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (el)->next = (add); \ + } \ + else { \ + LL_PREPEND2(head, add, next); \ + } \ + } while (0) + +#define LL_APPEND_ELEM(head, el, add) LL_APPEND_ELEM2(head, el, add, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef LL_CONCAT2 +#define LL_CONCAT2(head1, head2, next) \ + do { \ + char *_tmp; \ + if (head1) { \ + _tmp = (char *)(head1); \ + while ((head1)->next) { \ + (head1) = (head1)->next; \ + } \ + (head1)->next = (head2); \ + UTLIST_RS(head1); \ + } \ + else { \ + (head1) = (head2); \ + } \ + } while (0) + +#undef LL_APPEND2 +#define LL_APPEND2(head, add, next) \ + do { \ + if (head) { \ + (add)->next = head; /* use add->next as a temp variable */ \ + while ((add)->next->next) { \ + (add)->next = (add)->next->next; \ + } \ + (add)->next->next = (add); \ + } \ + else { \ + (head) = (add); \ + } \ + (add)->next = NULL; \ + } while (0) + +#undef LL_INSERT_INORDER2 +#define LL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, add)) >= 0) { \ + (add)->next = (head); \ + (head) = (add); \ + } \ + else { \ + char *_tmp = (char *)(head); \ + while ((head)->next != NULL && (cmp((head)->next, add)) < 0) { \ + (head) = (head)->next; \ + } \ + (add)->next = (head)->next; \ + (head)->next = (add); \ + UTLIST_RS(head); \ + } \ + } while (0) + +#undef LL_DELETE2 +#define LL_DELETE2(head, del, next) \ + do { \ + if ((head) == (del)) { \ + (head) = (head)->next; \ + } \ + else { \ + char *_tmp = (char *)(head); \ + while ((head)->next && ((head)->next != (del))) { \ + (head) = (head)->next; \ + } \ + if ((head)->next) { \ + (head)->next = ((del)->next); \ + } \ + UTLIST_RS(head); \ + } \ + } while (0) + +#undef LL_REPLACE_ELEM2 +#define LL_REPLACE_ELEM2(head, el, add, next) \ + do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + else { \ + (add)->next = head; \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el)->next; \ + } while (0) + +#undef LL_PREPEND_ELEM2 +#define LL_PREPEND_ELEM2(head, el, add, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + else { \ + (add)->next = (head); \ + while ((add)->next->next && ((add)->next->next != (el))) { \ + (add)->next = (add)->next->next; \ + } \ + if ((add)->next->next) { \ + (add)->next->next = (add); \ + } \ + } \ + (add)->next = (el); \ + } \ + else { \ + LL_APPEND2(head, add, next); \ + } \ + } while (0) + +#endif /* NO_DECLTYPE */ + +/****************************************************************************** + * doubly linked list macros (non-circular) * + *****************************************************************************/ +#define DL_PREPEND(head, add) DL_PREPEND2(head, add, prev, next) + +#define DL_PREPEND2(head, add, prev, next) \ + do { \ + (add)->next = (head); \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev = (add); \ + } \ + else { \ + (add)->prev = (add); \ + } \ + (head) = (add); \ + } while (0) + +#define DL_APPEND(head, add) DL_APPEND2(head, add, prev, next) + +#define DL_APPEND2(head, add, prev, next) \ + do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (head)->prev->next = (add); \ + (head)->prev = (add); \ + (add)->next = NULL; \ + } \ + else { \ + (head) = (add); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ + } while (0) + +#define DL_INSERT_INORDER(head, add, cmp) DL_INSERT_INORDER2(head, add, cmp, next) + +#define DL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if (head) { \ + DL_LOWER_BOUND(head, _tmp, add, cmp); \ + DL_APPEND_ELEM(head, _tmp, add); \ + } \ + else { \ + (head) = (add); \ + (head)->prev = (head); \ + (head)->next = NULL; \ + } \ + } while (0) + +#define DL_LOWER_BOUND(head, elt, like, cmp) DL_LOWER_BOUND2(head, elt, like, cmp, next) + +#define DL_LOWER_BOUND2(head, elt, like, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, like)) >= 0) { \ + (elt) = NULL; \ + } \ + else { \ + for ((elt) = (head); (elt)->next != NULL; (elt) = (elt)->next) { \ + if ((cmp((elt)->next, like)) >= 0) { \ + break; \ + } \ + } \ + } \ + } while (0) + +#define DL_CONCAT(head1, head2) DL_CONCAT2(head1, head2, prev, next) + +#define DL_CONCAT2(head1, head2, prev, next) \ + do { \ + LDECLTYPE(head1) _tmp; \ + if (head2) { \ + if (head1) { \ + UTLIST_CASTASGN(_tmp, (head2)->prev); \ + (head2)->prev = (head1)->prev; \ + (head1)->prev->next = (head2); \ + UTLIST_CASTASGN((head1)->prev, _tmp); \ + } \ + else { \ + (head1) = (head2); \ + } \ + } \ + } while (0) + +#define DL_DELETE(head, del) DL_DELETE2(head, del, prev, next) + +#define DL_DELETE2(head, del, prev, next) \ + do { \ + assert((head) != NULL); \ + assert((del)->prev != NULL); \ + if ((del)->prev == (del)) { \ + (head) = NULL; \ + } \ + else if ((del) == (head)) { \ + (del)->next->prev = (del)->prev; \ + (head) = (del)->next; \ + } \ + else { \ + (del)->prev->next = (del)->next; \ + if ((del)->next) { \ + (del)->next->prev = (del)->prev; \ + } \ + else { \ + (head)->prev = (del)->prev; \ + } \ + } \ + } while (0) + +#define DL_COUNT(head, el, counter) DL_COUNT2(head, el, counter, next) + +#define DL_COUNT2(head, el, counter, next) \ + do { \ + (counter) = 0; \ + DL_FOREACH2(head, el, next) \ + { \ + ++(counter); \ + } \ + } while (0) + +#define DL_FOREACH(head, el) DL_FOREACH2(head, el, next) + +#define DL_FOREACH2(head, el, next) for ((el) = (head); el; (el) = (el)->next) + +/* this version is safe for deleting the elements during iteration */ +#define DL_FOREACH_SAFE(head, el, tmp) DL_FOREACH_SAFE2(head, el, tmp, next) + +#define DL_FOREACH_SAFE2(head, el, tmp, next) \ + for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) + +/* these are identical to their singly-linked list counterparts */ +#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR +#define DL_SEARCH LL_SEARCH +#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2 +#define DL_SEARCH2 LL_SEARCH2 + +#define DL_REPLACE_ELEM2(head, el, add, prev, next) \ + do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((head) == (el)) { \ + (head) = (add); \ + (add)->next = (el)->next; \ + if ((el)->next == NULL) { \ + (add)->prev = (add); \ + } \ + else { \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + } \ + } \ + else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->prev->next = (add); \ + if ((el)->next == NULL) { \ + (head)->prev = (add); \ + } \ + else { \ + (add)->next->prev = (add); \ + } \ + } \ + } while (0) + +#define DL_REPLACE_ELEM(head, el, add) DL_REPLACE_ELEM2(head, el, add, prev, next) + +#define DL_PREPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + else { \ + (add)->prev->next = (add); \ + } \ + } \ + else { \ + DL_APPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define DL_PREPEND_ELEM(head, el, add) DL_PREPEND_ELEM2(head, el, add, prev, next) + +#define DL_APPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + if ((add)->next) { \ + (add)->next->prev = (add); \ + } \ + else { \ + (head)->prev = (add); \ + } \ + } \ + else { \ + DL_PREPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define DL_APPEND_ELEM(head, el, add) DL_APPEND_ELEM2(head, el, add, prev, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef DL_INSERT_INORDER2 +#define DL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + if ((head) == NULL) { \ + (add)->prev = (add); \ + (add)->next = NULL; \ + (head) = (add); \ + } \ + else if ((cmp(head, add)) >= 0) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (head) = (add); \ + } \ + else { \ + char *_tmp = (char *)(head); \ + while ((char *)(head)->next != _tmp && (cmp((head)->next, add)) < 0) { \ + (head) = (head)->next; \ + } \ + (add)->prev = (head); \ + (add)->next = (head)->next; \ + (head)->next = (add); \ + UTLIST_RS(head); \ + if ((add)->next) { \ + (add)->next->prev = (add); \ + } \ + else { \ + (head)->prev = (add); \ + } \ + } \ + } while (0) +#endif /* NO_DECLTYPE */ + +/****************************************************************************** + * circular doubly linked list macros * + *****************************************************************************/ +#define CDL_APPEND(head, add) CDL_APPEND2(head, add, prev, next) + +#define CDL_APPEND2(head, add, prev, next) \ + do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } \ + else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + (head) = (add); \ + } \ + } while (0) + +#define CDL_PREPEND(head, add) CDL_PREPEND2(head, add, prev, next) + +#define CDL_PREPEND2(head, add, prev, next) \ + do { \ + if (head) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (head)->prev = (add); \ + (add)->prev->next = (add); \ + } \ + else { \ + (add)->prev = (add); \ + (add)->next = (add); \ + } \ + (head) = (add); \ + } while (0) + +#define CDL_INSERT_INORDER(head, add, cmp) CDL_INSERT_INORDER2(head, add, cmp, next) + +#define CDL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + LDECLTYPE(head) _tmp; \ + if (head) { \ + CDL_LOWER_BOUND(head, _tmp, add, cmp); \ + CDL_APPEND_ELEM(head, _tmp, add); \ + } \ + else { \ + (head) = (add); \ + (head)->next = (head); \ + (head)->prev = (head); \ + } \ + } while (0) + +#define CDL_LOWER_BOUND(head, elt, like, cmp) CDL_LOWER_BOUND2(head, elt, like, cmp, next) + +#define CDL_LOWER_BOUND2(head, elt, like, cmp, next) \ + do { \ + if ((head) == NULL || (cmp(head, like)) >= 0) { \ + (elt) = NULL; \ + } \ + else { \ + for ((elt) = (head); (elt)->next != (head); (elt) = (elt)->next) { \ + if ((cmp((elt)->next, like)) >= 0) { \ + break; \ + } \ + } \ + } \ + } while (0) + +#define CDL_DELETE(head, del) CDL_DELETE2(head, del, prev, next) + +#define CDL_DELETE2(head, del, prev, next) \ + do { \ + if (((head) == (del)) && ((head)->next == (head))) { \ + (head) = NULL; \ + } \ + else { \ + (del)->next->prev = (del)->prev; \ + (del)->prev->next = (del)->next; \ + if ((del) == (head)) \ + (head) = (del)->next; \ + } \ + } while (0) + +#define CDL_COUNT(head, el, counter) CDL_COUNT2(head, el, counter, next) + +#define CDL_COUNT2(head, el, counter, next) \ + do { \ + (counter) = 0; \ + CDL_FOREACH2(head, el, next) \ + { \ + ++(counter); \ + } \ + } while (0) + +#define CDL_FOREACH(head, el) CDL_FOREACH2(head, el, next) + +#define CDL_FOREACH2(head, el, next) \ + for ((el) = (head); el; (el) = (((el)->next == (head)) ? NULL : (el)->next)) + +#define CDL_FOREACH_SAFE(head, el, tmp1, tmp2) CDL_FOREACH_SAFE2(head, el, tmp1, tmp2, prev, next) + +#define CDL_FOREACH_SAFE2(head, el, tmp1, tmp2, prev, next) \ + for ((el) = (head), (tmp1) = (head) ? (head)->prev : NULL; (el) && ((tmp2) = (el)->next, 1); \ + (el) = ((el) == (tmp1) ? NULL : (tmp2))) + +#define CDL_SEARCH_SCALAR(head, out, field, val) CDL_SEARCH_SCALAR2(head, out, field, val, next) + +#define CDL_SEARCH_SCALAR2(head, out, field, val, next) \ + do { \ + CDL_FOREACH2(head, out, next) \ + { \ + if ((out)->field == (val)) \ + break; \ + } \ + } while (0) + +#define CDL_SEARCH(head, out, elt, cmp) CDL_SEARCH2(head, out, elt, cmp, next) + +#define CDL_SEARCH2(head, out, elt, cmp, next) \ + do { \ + CDL_FOREACH2(head, out, next) \ + { \ + if ((cmp(out, elt)) == 0) \ + break; \ + } \ + } while (0) + +#define CDL_REPLACE_ELEM2(head, el, add, prev, next) \ + do { \ + assert((head) != NULL); \ + assert((el) != NULL); \ + assert((add) != NULL); \ + if ((el)->next == (el)) { \ + (add)->next = (add); \ + (add)->prev = (add); \ + (head) = (add); \ + } \ + else { \ + (add)->next = (el)->next; \ + (add)->prev = (el)->prev; \ + (add)->next->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } \ + } while (0) + +#define CDL_REPLACE_ELEM(head, el, add) CDL_REPLACE_ELEM2(head, el, add, prev, next) + +#define CDL_PREPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el); \ + (add)->prev = (el)->prev; \ + (el)->prev = (add); \ + (add)->prev->next = (add); \ + if ((head) == (el)) { \ + (head) = (add); \ + } \ + } \ + else { \ + CDL_APPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define CDL_PREPEND_ELEM(head, el, add) CDL_PREPEND_ELEM2(head, el, add, prev, next) + +#define CDL_APPEND_ELEM2(head, el, add, prev, next) \ + do { \ + if (el) { \ + assert((head) != NULL); \ + assert((add) != NULL); \ + (add)->next = (el)->next; \ + (add)->prev = (el); \ + (el)->next = (add); \ + (add)->next->prev = (add); \ + } \ + else { \ + CDL_PREPEND2(head, add, prev, next); \ + } \ + } while (0) + +#define CDL_APPEND_ELEM(head, el, add) CDL_APPEND_ELEM2(head, el, add, prev, next) + +#ifdef NO_DECLTYPE +/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ + +#undef CDL_INSERT_INORDER2 +#define CDL_INSERT_INORDER2(head, add, cmp, next) \ + do { \ + if ((head) == NULL) { \ + (add)->prev = (add); \ + (add)->next = (add); \ + (head) = (add); \ + } \ + else if ((cmp(head, add)) >= 0) { \ + (add)->prev = (head)->prev; \ + (add)->next = (head); \ + (add)->prev->next = (add); \ + (head)->prev = (add); \ + (head) = (add); \ + } \ + else { \ + char *_tmp = (char *)(head); \ + while ((char *)(head)->next != _tmp && (cmp((head)->next, add)) < 0) { \ + (head) = (head)->next; \ + } \ + (add)->prev = (head); \ + (add)->next = (head)->next; \ + (add)->next->prev = (add); \ + (head)->next = (add); \ + UTLIST_RS(head); \ + } \ + } while (0) +#endif /* NO_DECLTYPE */ + +#endif /* UTLIST_H */ \ No newline at end of file diff --git a/src/commons/collections/include/pdc_pht.h b/src/commons/collections/include/pdc_pht.h new file mode 100644 index 000000000..a4f139617 --- /dev/null +++ b/src/commons/collections/include/pdc_pht.h @@ -0,0 +1,96 @@ +#define ROOT_PREFIX "#" +typedef void* PrefixTableKey; +typedef void* PrefixTableValue; + +/** + * Hash function used to generate hash values for keys used in a hash + * table. + * + * @param data The data to generate a hash value for. + * @return The hash value. + */ + +typedef unsigned int (*PrefixTableHashFunc)(PrefixTableKey data); + +/** + * Function used to compare two keys for equality. + * + * @return Non-zero if the two keys are equal, zero if the keys are + * not equal. + */ + +typedef int (*PrefixTableEqualFunc)(PrefixTableKey value1, PrefixTableKey value2); + +/** + * Type of function used to free keys when entries are removed from a + * hash table. + */ + +typedef void (*PrefixTableKeyFreeFunc)(PrefixTableKey value); + +/** + * Prefix table data structure + */ +typedef struct _PrefixTable PrefixTable; + +/** + * Prefix table bucket structure + */ +typedef struct _PrefixTableBucket PrefixTableBucket; + +/** + * Initializes a new prefix table. + * + * @param bucket_size The size of each bucket in the prefix table. + * @param free_cb Function to free keys when they are removed from the table. + * @return A pointer to the initialized prefix table, or NULL on failure. + */ +PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb, PrefixTableKeyFreeFunc free_cb); + +/** + * Inserts a key-value pair into the prefix table. + * + * @param pht The prefix table to insert into. + * @param key The key to insert. + * @param value The value to associate with the key. + * @return 0 on success, or a negative error code on failure. + */ +int prefix_table_insert(PrefixTable *pht, void *key, void *value); + +/** + * Searches for a key in the prefix table. + * + * @param pht The prefix table to search. + * @param key The key to search for. + * @param result Pointer to store the found value, or NULL if not found. + * @return 0 on success, or a negative error code on failure. + */ +int prefix_table_exact_search(PrefixTable *pht, void *key, void **result); + +/** + * Searches for keys in a range within the prefix table. + * + * @param pht The prefix table to search. + * @param start_key The starting key of the range. + * @param include_start Whether to include the start key in the results. + * @param end_key The ending key of the range. + * @param include_end Whether to include the end key in the results. + * @param result Pointer to store the found values, or NULL if not found. + * @return 0 on success, or a negative error code on failure. + */ +int prefix_table_range_search(PrefixTable *pht, void *start_key, bool include_start, void *end_key, bool include_end, void **result); + +/** + * Counts the number of keys in the prefix table. + * + * @param pht The prefix table to count keys in. + * @return The number of keys in the prefix table. + */ +unsigned int prefix_table_count(PrefixTable *pht); + +/** + * Destroys the prefix table and frees all associated resources. + * + * @param pht The prefix table to destroy. + */ +void prefix_table_destroy(PrefixTable *pht); \ No newline at end of file diff --git a/src/commons/collections/include/pdc_set.h b/src/commons/collections/include/pdc_set.h index c6f77f20d..a6818b82e 100644 --- a/src/commons/collections/include/pdc_set.h +++ b/src/commons/collections/include/pdc_set.h @@ -181,6 +181,17 @@ int set_remove(Set *set, SetValue data); int set_query(Set *set, SetValue data); +/** + * Copy the contents of one set to another. + * + * @param target The set to copy values into. + * @param source The set to copy values from. + * @return Zero on success, or -1 if it was not possible to + * allocate memory for the new entries. + */ +int set_copy(Set *target, Set* source); + + /** * Retrieve the number of entries in a set * diff --git a/src/commons/collections/pdc_dllist.c b/src/commons/collections/pdc_dllist.c new file mode 100644 index 000000000..5cc4dc942 --- /dev/null +++ b/src/commons/collections/pdc_dllist.c @@ -0,0 +1,73 @@ +#include "pdc_dllist.h" +#include "pdc_set.h" +#include + +struct _DoublyLinkedListItem{ + void * key; + void * value; + struct _DoublyLinkedListItem *prev; + struct _DoublyLinkedListItem *next; +}; + +struct _DoublyLinkedList { + DoublyLinkedListItem * head; + DoublyLinkedListHashFunc hash_func; + DoublyLinkedListEqualFunc equal_func; + DoublyLinkedListKeyFreeFunc key_free_func; + unsigned int count; +}; + +DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func, DoublyLinkedListKeyFreeFunc key_free_func){ + DoublyLinkedList *list = malloc(sizeof(DoublyLinkedList)); + if (!list) { + return NULL; // Memory allocation failed + } + list->hash_func = hash_func; + list->equal_func = equal_func; + list->key_free_func = key_free_func; + list->head = NULL; + list->count = 0; + return list; +} + +int dllist_insert(DoublyLinkedList *list, void *key, void *value){ + DoublyLinkedListItem *item = malloc(sizeof(DoublyLinkedListItem)); + item->key = key; + item->value = value; + DL_APPEND(list->head, item); + list->count++; + return 0; +} + +int *dllist_search_key(DoublyLinkedList *list, void *key){ + DoublyLinkedListItem *elt = NULL; + DoublyLinkedListEqualFunc key_cmp_func = list->equal_func; + DL_FOREACH(list->head, elt) { + if(key_cmp_func(elt->key, key) == 0) break; + } + return elt != NULL && key_cmp_func(elt->key, key) == 0 ? elt->value : NULL; +} + +int *dllist_search_range(DoublyLinkedList * list, void *start_key, bool include_start, void *end_key, bool include_end){ + DoublyLinkedListItem *elt = NULL; + Set *value_set = set_new(NULL, NULL); + DoublyLinkedListEqualFunc key_cmp_func = list->equal_func; + DL_FOREACH(list->head, elt) { + int start_dist = key_cmp_func(elt->key, start_key); + int end_dist = key_cmp_func(elt->key, end_key); + if(start_dist >= 0 && end_dist <= 0){ + set_copy(value_set, elt->value); + } + } + return value_set; +} + +int dllist_destroy(void *ptr){ + DoublyLinkedList *list = (DoublyLinkedList*)ptr; + DoublyLinkedListItem *elt = NULL; + DoublyLinkedListItem *tmp = NULL; + DL_FOREACH_SAFE(list->head, elt, tmp) { + DL_DELETE(list->head, elt); + free(elt); + } +} \ No newline at end of file diff --git a/src/commons/collections/pdc_pht.c b/src/commons/collections/pdc_pht.c new file mode 100644 index 000000000..7633dcb3b --- /dev/null +++ b/src/commons/collections/pdc_pht.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include +#include "pdc_hash_table.h" +#include "pdc_pht.h" +#include "pdc_dllist.h" + +#ifdef __cplusplus +extern "C" { +#endif +struct _PrefixTableBucket { + bool isLeaf; // true if this is a leaf node + /** + * Parent and siblings + */ + char *parent; + char *left; + char *right; + + char *prefix; + DoublyLinkedList *store; +}; + +struct _PrefixTable { + HashTable *map; + size_t bucket_size; + size_t key_count; +}; + +PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb, PrefixTableKeyFreeFunc free_cb){ + PrefixTable *pht = (PrefixTable *)malloc(sizeof(PrefixTable)); + if (!pht) { + return NULL; // Memory allocation failed + } + pht->bucket_size = bucket_size; + pht->key_count = 0; + pht->map = hash_table_new(hash_cb, equal_cb); + if (!pht->map) { + free(pht); + return NULL; // Hash table creation failed + } + return pht; +} + +PrefixTableBucket* prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb, PrefixTableKeyFreeFunc free_cb){ + PrefixTableBucket *bucket = (PrefixTableBucket *)malloc(sizeof(PrefixTableBucket)); + if (!bucket) { + return NULL; // Memory allocation failed + } + bucket->isLeaf = false; + bucket->parent = NULL; + bucket->left = NULL; + bucket->right = NULL; + bucket->prefix = NULL; + bucket->store = dllist_init(hash_cb, equal_cb, free_cb); + return bucket; +} + +void prefix_table_destroy(PrefixTable *pht){ + hash_table_free(pht->map); + free(pht); +} + +void pht_add_to_store(PrefixTable *pht, void *key, void *value){ + hash_table_insert(pht->map, key, value); +} + + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/src/commons/collections/pdc_set.c b/src/commons/collections/pdc_set.c index 47f7fb35d..1f46de85a 100644 --- a/src/commons/collections/pdc_set.c +++ b/src/commons/collections/pdc_set.c @@ -405,6 +405,30 @@ set_num_entries(Set *set) FUNC_LEAVE(set->entries); } +int set_copy(Set *target, Set* source){ + SetIterator iterator; + SetValue value; + set_iterate(source, &iterator); + + while (set_iter_has_more(&iterator)) { + + /* Read the next value */ + + value = set_iter_next(&iterator); + /* Has this value been put into the new set already? + * If so, do not insert this again */ + + if (set_query(target, value) == 0) { + if (!set_insert(target, value)) { + /* Failed to insert */ + set_free(target); + return -1; + } + } + } + return 0; +} + SetValue * set_to_array(Set *set) { diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index 92da04254..0376c8523 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -517,6 +517,14 @@ typedef struct { int32_t ret; } metadata_delete_by_id_out_t; +typedef struct { + char* prefix; +} metadata_check_prefix_in_t; + +typedef struct { + int32_t ret; +} metadata_check_prefix_out_t; + /* Define region_lock_out_t */ typedef struct { int32_t ret; @@ -1728,6 +1736,16 @@ hg_proc_metadata_add_tag_in_t(hg_proc_t proc, void *data) FUNC_LEAVE(ret); } +static HG_INLINE hg_return_t +hg_proc_metadata_check_prefix_in_t(hg_proc_t proc, void *data) { + return HG_SUCCESS; +} + +static HG_INLINE hg_return_t +hg_proc_metadata_check_prefix_out_t(hg_proc_t proc, void *data) { + return HG_SUCCESS; +} + /* Define hg_proc_metadata_get_kvtag_in_t */ static HG_INLINE hg_return_t hg_proc_metadata_get_kvtag_in_t(hg_proc_t proc, void *data) @@ -4204,6 +4222,7 @@ hg_id_t PDC_metadata_delete_by_id_register(hg_class_t *hg_class); hg_id_t PDC_metadata_update_register(hg_class_t *hg_class); hg_id_t PDC_metadata_add_tag_register(hg_class_t *hg_class); hg_id_t PDC_metadata_add_kvtag_register(hg_class_t *hg_class); +hg_id_t PDC_metadata_check_prefix_register(hg_class_t *hg_class); hg_id_t PDC_metadata_del_kvtag_register(hg_class_t *hg_class); hg_id_t PDC_metadata_get_kvtag_register(hg_class_t *hg_class); hg_id_t PDC_send_rpc_register(hg_class_t *hg_class); @@ -4753,6 +4772,24 @@ perr_t PDC_create_shm_segment(region_list_t *region); */ perr_t PDC_create_shm_segment_ind(uint64_t size, char *shm_addr, void **buf); +/** + * Convert a string to binary + * + * \param ptr[IN] Pointer to the string + * + * \return Binary representation of the string + */ +char* string_to_binary(void *ptr); + +/** + * Calculate the hash value of a string + * + * \param key[IN] String to be hashed + * + * \return Hash value of the string + */ +uint32_t prefix_hash(const char *key); + /** * Duplicate a kvtag * diff --git a/src/server/include/pdc_server_metadata.h b/src/server/include/pdc_server_metadata.h index cb763d2e6..87291f90c 100644 --- a/src/server/include/pdc_server_metadata.h +++ b/src/server/include/pdc_server_metadata.h @@ -380,4 +380,14 @@ perr_t PDC_free_cont_hash_table(); */ perr_t PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out); +/** + * Check if there is any object with the given prefix + * + * \param in [IN] Input structure received from client + * \param out [IN] Output structure to be sent back to the client + * + * \return Non-negative on success/Negative on failure + */ +perr_t PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out); + #endif /* PDC_SERVER_METADATA_H */ diff --git a/src/server/include/pdc_server_metadata_pht.h b/src/server/include/pdc_server_metadata_pht.h new file mode 100644 index 000000000..e69de29bb diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index b75c8f567..d4c534e4d 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -224,6 +224,46 @@ PDC_get_server_by_obj_id(uint64_t obj_id, int n_server) FUNC_LEAVE(ret_value); } +uint32_t prefix_hash(const char *key){ + uint32_t hash = 0; + int i = 1; + while (*key) { + if (*key == '\0') break; + if (*key == '#') { + *key++; + continue; + }; + if (*key == '0') { + hash = (hash*193) + (i * 90); + } else { + hash = (hash*193) + (i * 270); + } + *key++; + i++; + } + return hash; +} + +char* string_to_binary(void *ptr){ + char *str = (char *)ptr; + size_t len = strlen(str); + char *binStr = malloc(len*8 + 1); + for (int i = 0; i < len; i++){ + char c = str[i]; + for (int j=7; j>=0; j--) { + binStr[(i*8) + 7 - j] = (c & (1 << j)) ? '1' : '0'; + } + } + binStr[len*8] = '\0'; + return binStr; +} + +uint32_t +PDC_get_server_by_obj_id_pht(uint64_t obj_id, int n_server) +{ + +} + int PDC_get_var_type_size(pdc_var_type_t dtype) { @@ -965,6 +1005,13 @@ PDC_Server_get_kvtag(metadata_get_kvtag_in_t *in ATTRIBUTE(unused), FUNC_ENTER(NULL); FUNC_LEAVE(SUCCEED); } + +perr_t +PDC_Server_check_prefix(metadata_check_prefix_in_t *in ATTRIBUTE(unused), + metadata_check_prefix_out_t *out ATTRIBUTE(unused)) +{ + return SUCCEED; +} perr_t PDC_Meta_Server_buf_unmap(buf_unmap_in_t *in ATTRIBUTE(unused), hg_handle_t *handle ATTRIBUTE(unused)) { @@ -1844,6 +1891,31 @@ HG_TEST_RPC_CB(metadata_add_kvtag, handle) FUNC_LEAVE(ret_value); } +HG_TEST_RPC_CB(metadata_check_prefix, handle) +{ + FUNC_ENTER(NULL); + + hg_return_t ret_value = HG_SUCCESS; + metadata_check_prefix_in_t in; + metadata_check_prefix_out_t out; + + HG_Get_input(handle, &in); + if (strcmp(in.prefix, "PDC_NOOP") != 0) { + PDC_Server_check_prefix(&in, &out); + } + else { + LOG_INFO("Received NOOP\n"); + out.ret = 1; + } + + ret_value = HG_Respond(handle, NULL, NULL, &out); + + HG_Free_input(handle, &in); + HG_Destroy(handle); + + FUNC_LEAVE(ret_value); +} + /* static hg_return_t */ // notify_io_complete_cb(hg_handle_t handle) HG_TEST_RPC_CB(notify_io_complete, handle) @@ -6160,6 +6232,7 @@ PDC_FUNC_DECLARE_REGISTER(metadata_add_tag) PDC_FUNC_DECLARE_REGISTER(send_rpc) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_del_kvtag, metadata_get_kvtag_in_t, metadata_add_tag_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_add_kvtag, metadata_add_kvtag_in_t, metadata_add_tag_out_t) +PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_check_prefix, metadata_check_prefix_in_t, metadata_check_prefix_out_t) PDC_FUNC_DECLARE_REGISTER(metadata_get_kvtag) PDC_FUNC_DECLARE_REGISTER(metadata_update) PDC_FUNC_DECLARE_REGISTER(metadata_delete_by_id) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index ffb967dbd..b8b405d84 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -147,6 +147,7 @@ int gen_fastbit_idx_g = 0; int use_fastbit_idx_g = 0; int use_rocksdb_g = 0; int use_sqlite3_g = 0; +int use_pht_g = 0; char * gBinningOption = NULL; double server_write_time_g = 0.0; @@ -1928,6 +1929,7 @@ PDC_Server_mercury_register() PDC_region_release_register(hg_class_g); PDC_gen_cont_id_register(hg_class_g); PDC_metadata_add_kvtag_register(hg_class_g); + PDC_metadata_check_prefix_register(hg_class_g); PDC_metadata_get_kvtag_register(hg_class_g); PDC_metadata_del_kvtag_register(hg_class_g); PDC_send_rpc_register(hg_class_g); diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index efec78ff5..05ef6d5dd 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -1815,6 +1815,10 @@ PDC_Server_query_kvtag_someta(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t **obj_ FUNC_LEAVE(ret_value); } +static perr_t +PDC_Server_query_kvtag_pht(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t **obj_ids, uint64_t alloc_size) +{ +} perr_t PDC_Server_get_kvtag_query_result(pdc_kvtag_t *in /*FIXME: query input should be string-based*/, uint32_t *n_meta, uint64_t **obj_ids) @@ -2739,6 +2743,18 @@ PDC_Server_add_kvtag_someta(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t FUNC_LEAVE(ret_value); } +/*** + * This function will build prefix hash table for this particular server + * TODO: Need to register using Mercury RPC + * \param in[IN] Input structure received from client + * \param out[OUT] Output structure to be sent back to the client + */ + +perr_t +PDC_Server_add_metadata_key(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) +{ + +} perr_t PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) { @@ -2812,6 +2828,66 @@ PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) FUNC_LEAVE(ret_value); } +perr_t +PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out) +{ + perr_t ret_value = SUCCEED; +#ifdef ENABLE_MULTITHREAD + int unlocked; +#endif + FUNC_ENTER(NULL); + +#ifdef ENABLE_TIMING + struct timeval pdc_timer_start; + struct timeval pdc_timer_end; + double ht_total_sec; + gettimeofday(&pdc_timer_start, 0); +#endif + + out->ret = -1; + +#ifdef ENABLE_MULTITHREAD + // Obtain lock for hash table + unlocked = 0; + hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); +#endif + + out->ret = 0; + +done: +#ifdef ENABLE_MULTITHREAD + // ^ Release hash table lock + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); + unlocked = 1; +#endif + +#ifdef ENABLE_TIMING + // Timing + gettimeofday(&pdc_timer_end, 0); + ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_TIMING + server_update_time_g += ht_total_sec; +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_unlock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_MULTITHREAD + if (unlocked == 0) + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); +#endif + fflush(stdout); + + FUNC_LEAVE(ret_value); +} + static perr_t PDC_get_kvtag_value_from_list(pdc_kvtag_list_t **list_head, char *key, metadata_get_kvtag_out_t *out) { @@ -2984,6 +3060,10 @@ PDC_Server_get_kvtag_someta(metadata_get_kvtag_in_t *in, metadata_get_kvtag_out_ FUNC_LEAVE(ret_value); } +static perr_t +PDC_Server_get_kvtag_pht(metadata_get_kvtag_in_t *in, metadata_get_kvtag_out_t *out) +{ +} perr_t PDC_Server_get_kvtag(metadata_get_kvtag_in_t *in, metadata_get_kvtag_out_t *out) diff --git a/src/server/pdc_server_metadata_pht.c b/src/server/pdc_server_metadata_pht.c new file mode 100644 index 000000000..87b3e9385 --- /dev/null +++ b/src/server/pdc_server_metadata_pht.c @@ -0,0 +1,162 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pdc_config.h" + +#ifdef ENABLE_MPI +#include "mpi.h" +#endif + +#include "pdc_utlist.h" +#include "pdc_hash_table.h" +#include "pdc_dablooms.h" +#include "pdc_interface.h" +#include "pdc_client_server_common.h" +#include "pdc_server_metadata_pht.h" +#include "pdc_server.h" +#include "mercury_hash_table.h" +#include "pdc_malloc.h" +#include "string_utils.h" +#include "pdc_logger.h" +#include "pdc_pht.h" + + +PrefixTable *metadata_pht_g = NULL; + +/* + * Check if two hash keys are equal + * + * \param vlocation1 [IN] Hash table key + * \param vlocation2 [IN] Hash table key + * + * \return 1 if two keys are equal, 0 otherwise + */ +static int +PDC_Server_metadata_pht_int_compare(void *vlocation1, void *vlocation2) +{ + int32_t k1 = *(int32_t *) key1; + int32_t k2 = *(int32_t *) key2; + + return (k1 > k2) - (k1 < k2); +} + +/* + * Get hash key's location in hash table + * + * \param vlocation [IN] Hash table key + * + * \return the location of hash key in the table + */ +static unsigned int +PDC_Server_metadata_pht_int_hash(void *vlocation) +{ + int *location; + location = (int *) vlocation; + return (unsigned int) *location; +} + +/* + * Free the hash key + * + * \param key [IN] Hash table key + * + * \return void + */ +static void +PDC_Server_metadata_pht_int_hash_key_free(void *key) +{ + free(key); +} + +/* + * Free metadata hash value + * + * \param value [IN] Hash table value + * + * \return void + */ +static void +PDC_Server_metadata_pht_value_free(void *value) +{ + pdc_metadata_t *elt, *tmp; + PrefixTableBucket *head; + + FUNC_ENTER(NULL); + + head = (PrefixTableBucket *)value; + + // Free metadata list + if (is_restart_g == 0) { + DL_FOREACH_SAFE(head.store, elt, tmp) + { + free(elt); + } + } +} + +/* + * Get the metadata with obj ID from the metadata list + * + * \param mlist[IN] Metadata list head + * \param obj_id[IN] Object ID + * + * \return NULL if no match is found/pointer to the found metadata otherwise + */ +static pdc_metadata_t * +find_metadata_pht_by_id_from_list(pdc_metadata_t *mlist, uint64_t obj_id) +{ + pdc_metadata_t *ret_value, *elt; + + FUNC_ENTER(NULL); + + ret_value = NULL; + if (mlist == NULL) { + ret_value = NULL; + goto done; + } + + DL_FOREACH(mlist, elt) + { + if (elt->obj_id == obj_id) { + ret_value = elt; + goto done; + } + } + +done: + FUNC_LEAVE(ret_value); +} \ No newline at end of file From 82ef1383a0ccc39ccc418b2053e669649486167c Mon Sep 17 00:00:00 2001 From: Suben Saha Date: Mon, 6 Oct 2025 07:46:14 -0700 Subject: [PATCH 2/3] added PHT insert, delete --- src/api/include/pdc_client_connect.h | 7 + src/api/pdc_client_connect.c | 286 ++++++++++-- src/commons/collections/include/pdc_dllist.h | 30 +- src/commons/collections/include/pdc_pht.h | 80 +++- .../collections}/include/pdc_utlist.h | 0 src/commons/collections/pdc_dllist.c | 17 +- src/commons/collections/pdc_pht.c | 46 +- src/server/CMakeLists.txt | 1 + src/server/include/pdc_client_server_common.h | 137 +++++- src/server/include/pdc_server.h | 4 +- src/server/include/pdc_server_metadata.h | 10 +- src/server/include/pdc_server_metadata_pht.h | 29 ++ src/server/pdc_client_server_common.c | 97 +++- src/server/pdc_server.c | 71 ++- src/server/pdc_server_metadata.c | 64 +-- src/server/pdc_server_metadata_pht.c | 414 ++++++++++++++---- src/tests/CMakeLists.txt | 8 +- src/tests/pht/pht_add_key.c | 89 ++++ src/tests/pht/pht_create_bucket.c | 55 +++ 19 files changed, 1177 insertions(+), 268 deletions(-) rename src/{server => commons/collections}/include/pdc_utlist.h (100%) create mode 100644 src/tests/pht/pht_add_key.c create mode 100644 src/tests/pht/pht_create_bucket.c diff --git a/src/api/include/pdc_client_connect.h b/src/api/include/pdc_client_connect.h index e8999ab2f..72e7ea9f9 100644 --- a/src/api/include/pdc_client_connect.h +++ b/src/api/include/pdc_client_connect.h @@ -1140,4 +1140,11 @@ void report_avg_server_profiling_rst(); perr_t PDC_Client_transfer_pthread_create(); perr_t PDC_Client_transfer_pthread_terminate(); perr_t PDC_Client_transfer_pthread_cnt_add(int n); + +perr_t PDC_Client_create_bucket(char *prefix); +uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix); +perr_t PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); +perr_t PDC_metadata_key_delete(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); +perr_t PDC_delete_metadata_key(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); +hg_return_t metadata_create_bucket_client_rpc_cb(const struct hg_cb_info *callback_info); #endif /* PDC_CLIENT_CONNECT_H */ diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index 81115ba6b..d63ec6dc9 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -70,7 +70,7 @@ #include "errno.h" /* #define TANG_DEBUG 1 */ -#define RING_ARC_SIZE 64 +#define ROOT_BUCKET "#" int is_client_debug_g = 0; pdc_server_selection_t pdc_server_selection_g = PDC_SERVER_DEFAULT; @@ -149,6 +149,7 @@ static hg_id_t metadata_update_register_id_g; static hg_id_t metadata_add_tag_register_id_g; static hg_id_t metadata_add_kvtag_register_id_g; static hg_id_t metadata_check_prefix_register_id_g; +static hg_id_t metadata_key_add_register_id_g; static hg_id_t metadata_del_kvtag_register_id_g; static hg_id_t metadata_get_kvtag_register_id_g; static hg_id_t region_lock_register_id_g; @@ -1347,6 +1348,8 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po metadata_add_tag_register_id_g = PDC_metadata_add_tag_register(*hg_class); metadata_add_kvtag_register_id_g = PDC_metadata_add_kvtag_register(*hg_class); metadata_check_prefix_register_id_g = PDC_metadata_check_prefix_register(*hg_class); + metadata_create_bucket_register_id_g = PDC_metadata_create_bucket_register(*hg_class); + metadata_key_add_register_id_g = PDC_metadata_key_add_register(*hg_class); metadata_del_kvtag_register_id_g = PDC_metadata_del_kvtag_register(*hg_class); metadata_get_kvtag_register_id_g = PDC_metadata_get_kvtag_register(*hg_class); region_lock_register_id_g = PDC_region_lock_register(*hg_class); @@ -1550,8 +1553,9 @@ PDC_Client_init() } if (pdc_client_mpi_rank_g == 0) { - LOG_INFO("Using [%s] as tmp dir, %d clients per server\n", pdc_client_tmp_dir_g, + LOG_INFO("Using [%s] as tmp dir112, %d clients per server\n", pdc_client_tmp_dir_g, pdc_nclient_per_server_g); + PDC_Client_create_bucket(ROOT_BUCKET); } if (mercury_has_init_g) { @@ -2046,6 +2050,68 @@ metadata_add_tag_rpc_cb(const struct hg_cb_info *callback_info) FUNC_LEAVE(ret_value); } +static hg_return_t +metadata_check_prefix_rpc_cb(const struct hg_cb_info *callback_info) +{ + FUNC_ENTER(NULL); + + hg_return_t ret_value; + metadata_check_prefix_out_t output; + metadata_check_prefix_out_t *args = (metadata_check_prefix_out_t *)callback_info->arg; + hg_handle_t handle = callback_info->info.forward.handle; + + printf("metadata_check_prefix_rpc_cb: callback_info->arg = %p\n", callback_info->arg); + /* Get output from server*/ + ret_value = HG_Get_output(handle, &output); + printf("metadata_check_prefix_rpc_cb: after HG_Get_output\n"); + if (ret_value != HG_SUCCESS) { + ret_value = -1; + PGOTO_ERROR(HG_OTHER_ERROR, "Error with HG_Get_output"); + } + args->found = output.found; + args->server_id = output.server_id; + args->ret = output.ret; + args->leaf = output.leaf; + printf("metadata_check_prefix_rpc_cb: output.found = %d\n", args->found); + printf("metadata_check_prefix_rpc_cb: output.leaf = %d\n", args->leaf); + printf("metadata_check_prefix_rpc_cb: output.ret = %d\n", args->ret); + printf("metadata_check_prefix_rpc_cb: output.server_id = %d\n", args->server_id); + +done: + hg_atomic_decr32(&atomic_work_todo_g); + HG_Free_output(handle, &output); + + FUNC_LEAVE(ret_value); +} + +static hg_return_t +metadata_key_add_rpc_cb(const struct hg_cb_info *callback_info) +{ + FUNC_ENTER(NULL); + + hg_return_t ret_value; + metadata_key_add_out_t output; + metadata_key_add_out_t *args = (metadata_key_add_out_t *)callback_info->arg; + hg_handle_t handle = callback_info->info.forward.handle; + + printf("metadata_key_add_rpc_cb: callback_info->arg = %p\n", callback_info->arg); + /* Get output from server*/ + ret_value = HG_Get_output(handle, &output); + printf("metadata_key_add_rpc_cb: after HG_Get_output\n"); + if (ret_value != HG_SUCCESS) { + ret_value = -1; + PGOTO_ERROR(HG_OTHER_ERROR, "Error with HG_Get_output"); + } + args->ret = output.ret; + printf("metadata_check_prefix_rpc_cb: output.ret = %d\n", args->ret); + +done: + hg_atomic_decr32(&atomic_work_todo_g); + HG_Free_output(handle, &output); + + FUNC_LEAVE(ret_value); +} + perr_t PDC_Client_add_tag(pdcid_t obj_id, const char *tag) { @@ -6363,6 +6429,7 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) } server_id = PDC_get_server_by_obj_id(meta_id, pdc_server_num_g); + printf("Object ID %" PRIu64 " mapped to server %" PRIu32 "\n", obj_id, server_id); // Debug statistics for counting number of messages sent to each server. debug_server_id_count[server_id]++; @@ -6409,65 +6476,149 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) */ static perr_t -PDC_check_prefix(uint32_t server, char *prefix){ +PDC_check_prefix(uint32_t server, char *prefix, metadata_check_prefix_out_t *out) { + FUNC_ENTER(NULL); + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + hg_handle_t metadata_check_prefix_handle; + struct _pdc_client_lookup_args lookup_args; + metadata_check_prefix_in_t in; -} + if (PDC_Client_try_lookup_server(server, 0) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); -uint32_t -PDC_get_server_using_pht(uint32_t key_hash) { - int ring_size = pdc_server_num_g * RING_ARC_SIZE; - uint64_t mapped_hash = key_hash % ring_size; + HG_Create(send_context_g, pdc_server_info_g[server].addr, metadata_check_prefix_register_id_g, + &metadata_check_prefix_handle); - for (int i = 0; i < pdc_server_num_g; i++) { - int node_pos = (i + 1) * ceil(ring_size / (float)pdc_server_num_g); + // Fill input structure + in.prefix = prefix; + printf("Checking prefix client %s on server %d\n", prefix, server); + hg_ret = HG_Forward(metadata_check_prefix_handle, metadata_check_prefix_rpc_cb, out, &in); + if (hg_ret != HG_SUCCESS) + PGOTO_ERROR(FAIL, "Could not start HG_Forward"); - if (mapped_hash <= node_pos) { - return i; - } - } + // Wait for response from server + hg_atomic_set32(&atomic_work_todo_g, 1); + PDC_Client_check_response(&send_context_g); - // Wrap around - return 0; + if (out->ret != 1) + LOG_ERROR("Check prefix NOT successful, ret_value = %d\n", out->ret); + +done: + HG_Destroy(metadata_check_prefix_handle); + + FUNC_LEAVE(ret_value); } -uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix, bool is_leaf){ - int target_server = -1; +uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix){ + metadata_check_prefix_out_t resp; + uint64_t temp_server_id = -1; int max = strlen(prefix) - 1; int mid = 1; char *slice; - + + printf("Searching prefix %s\n", prefix); while (mid <= max){ slice = malloc((mid + 1) * sizeof(char)); strncpy(slice, prefix, mid); slice[mid] = '\0'; - char *prefix_bin = string_to_binary(slice); - uint32_t hash_value = prefix_hash(prefix_bin); - target_server = PDC_get_server_using_pht(hash_value); - //void *resp = PDC_check_if_prefix_exists(target_server, slice); - - /** TODO: Check if the prefix bucket is redirected, make another call here */ - - *target_prefix = malloc((mid + 1) * sizeof(char)); - strncpy(*target_prefix, slice, mid); - (*target_prefix)[mid] = '\0'; + uint64_t hash_value = prefix_hash(prefix); + temp_server_id = PDC_get_server_using_pht(hash_value); + printf("Server %d is responsible for prefix %s till length %d\n", temp_server_id, slice, mid); + perr_t ret = PDC_check_prefix(temp_server_id, slice, &resp); + + printf("Checking prefix %s on server %d: found=%d, leaf=%d\n", slice, temp_server_id, resp.found, resp.leaf); + //TODO: check if redirect to another server /*** If the prefix exists and a leaf node, return the target server */ - + if (resp.found == 1 && resp.leaf == 1){ + *target_prefix = malloc((mid + 1) * sizeof(char)); + strncpy(*target_prefix, slice, mid); + (*target_prefix)[mid] = '\0'; + free(slice); + return temp_server_id; + } + else if (resp.found == 1 && resp.leaf == 0){ /** If the prefix exists but not a leaf node, search longer prefix */ + mid++; + } + else{ /** If the prefix does not exist, search shorter prefix */ + max = mid - 1; + mid = mid / 2; + } free(slice); - return target_server; + return temp_server_id; } + return -1; // No valid prefix found +} +perr_t PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { + FUNC_ENTER(NULL); + char *key_name; + char *target_prefix; + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + hg_handle_t metadata_key_add_handle; + metadata_key_add_in_t in; + struct _pdc_client_lookup_args lookup_args; + + printf("PDC_metadata_key_add: Adding key %s to metadata\n", kvtag->name); + key_name = kvtag->name; + char * key_bin = string_to_binary(key_name); + char *prefix = malloc(strlen(key_bin) + 2); + strncpy(prefix, ROOT_BUCKET, 1); + strncpy(prefix+1, key_bin, strlen(key_bin)); + prefix[strlen(key_bin)+1] = '\0'; + free(key_bin); + + uint32_t server_id = PDC_prefix_binary_search(prefix, &target_prefix); + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); + + HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_key_add_register_id_g, &metadata_key_add_handle); + + in.key = strdup(kvtag->name); + in.value = kvtag->value; + in.prefix = strdup(target_prefix); + in.size = kvtag->size; + + printf("PDC_metadata_key_add: Inserting key %s with prefix %s and value %s to server %d\n", in.key, in.prefix, in.value, server_id); + + hg_ret = HG_Forward(metadata_key_add_handle, metadata_key_add_rpc_cb, &lookup_args, &in); + printf("HG_Forward returned %d\n", hg_ret); + if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not start HG_Forward"); + + // Wait for response from server + hg_atomic_set32(&atomic_work_todo_g, 1); + PDC_Client_check_response(&send_context_g); + + free(target_prefix); + free(prefix); +done: + HG_Destroy(metadata_key_add_handle); + + FUNC_LEAVE(ret_value); } -static perr_t -PDC_add_metadata_key(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { - char *prefix; +perr_t PDC_metadata_key_delete(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { + FUNC_ENTER(NULL); char *key_name; + char *target_prefix; perr_t ret_value = SUCCEED; + printf("PDC_metadata_key_delete: Adding key %s to metadata\n", kvtag->name); key_name = kvtag->name; - prefix = string_to_binary(key_name); + char * key_bin = string_to_binary(key_name); + char *prefix = malloc(strlen(key_bin) + 2); + strncpy(prefix, ROOT_BUCKET, 1); + strncpy(prefix+1, key_bin, strlen(key_bin)); + prefix[strlen(key_bin)+1] = '\0'; + free(key_bin); + uint32_t server_id = PDC_prefix_binary_search(prefix, &target_prefix); + printf("PDC_metadata_key_delete: Inserting key %s with prefix %s to server %d\n", key_name, target_prefix, server_id); + free(target_prefix); + free(prefix); + FUNC_LEAVE(ret_value); } static hg_return_t @@ -8520,6 +8671,71 @@ PDC_Client_search_obj_ref_through_dart_mpi(dart_hash_algo_t hash_algo, char *que *out = dart_out; FUNC_LEAVE(ret_value); } + +perr_t +PDC_Client_create_bucket(char *prefix){ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + hg_handle_t metadata_create_bucket_handle; + metadata_create_bucket_in_t in; + struct _pdc_client_lookup_args lookup_args; + + uint64_t hash_value = prefix_hash(prefix); + uint32_t server_id = PDC_get_server_using_pht(hash_value); + + if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); + + HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_create_bucket_register_id_g, + &metadata_create_bucket_handle); + + // Fill input structure + in.prefix = prefix; + + hg_ret = HG_Forward(metadata_create_bucket_handle, metadata_create_bucket_client_rpc_cb, &lookup_args, &in); + if (hg_ret != HG_SUCCESS) + PGOTO_ERROR(FAIL, "Could not start HG_Forward"); + // Wait for response from server + hg_atomic_set32(&atomic_work_todo_g, 1); + PDC_Client_check_response(&send_context_g); // BLOCKING CALL + + if (lookup_args.ret != 1) + LOG_ERROR("Add create_bucket NOT successful, ret_value = %d\n", lookup_args.ret); +done: + HG_Destroy(metadata_create_bucket_handle); + + FUNC_LEAVE(ret_value); +} + +hg_return_t +metadata_create_bucket_client_rpc_cb(const struct hg_cb_info *callback_info) +{ + + FUNC_ENTER(NULL); + hg_return_t ret_value; + hg_handle_t handle = callback_info->info.forward.handle; + struct _pdc_client_lookup_args *client_lookup_args = (struct _pdc_client_lookup_args *)callback_info->arg; + /* Get output from server*/ + + metadata_create_bucket_out_t output; + ret_value = HG_Get_output(handle, &output); + + if (ret_value != HG_SUCCESS) { + client_lookup_args->ret = -1; + PGOTO_ERROR(HG_OTHER_ERROR, "Error with HG_Get_output"); + } + printf("metadata_create_bucket_rpc_cb: output.ret = %d\n", output.ret); + client_lookup_args->ret = output.ret; +done: + + hg_atomic_decr32(&atomic_work_todo_g); + HG_Free_output(handle, &output); + FUNC_LEAVE(ret_value); +} + + #endif /******************** Collective Object Selection Query Ends *******************************/ diff --git a/src/commons/collections/include/pdc_dllist.h b/src/commons/collections/include/pdc_dllist.h index c6cef2774..34022beef 100644 --- a/src/commons/collections/include/pdc_dllist.h +++ b/src/commons/collections/include/pdc_dllist.h @@ -54,38 +54,20 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * DL_APPEND(list, item); * } **/ + +// Function pointer typedefs typedef void* DoublyLinkedListKey; typedef void* DoublyLinkedListValue; -typedef struct _DoublyLinkedList DoublyLinkedList; -typedef struct _DoublyLinkedListItem DoublyLinkedListItem; - -/** - * Hash function used to generate hash values for keys used in a hash - * table. - * - * @param data The data to generate a hash value for. - * @return The hash value. - */ typedef unsigned int (*DoublyLinkedListHashFunc)(DoublyLinkedListKey data); - -/** - * Function used to compare two keys for equality. - * - * @return Non-zero if the two keys are equal, zero if the keys are - * not equal. - */ - typedef int (*DoublyLinkedListEqualFunc)(DoublyLinkedListKey value1, DoublyLinkedListKey value2); +typedef void (*DoublyLinkedListKeyFreeFunc)(DoublyLinkedListKey value); -/** - * Type of function used to free keys when entries are removed from a - * hash table. - */ +typedef struct DoublyLinkedListItem DoublyLinkedListItem; +typedef struct DoublyLinkedList DoublyLinkedList; -typedef void (*DoublyLinkedListKeyFreeFunc)(DoublyLinkedListKey value); -DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func, DoublyLinkedListKeyFreeFunc key_free_func); +DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func); int dllist_insert(DoublyLinkedList *list, void *key, void *value); int *dllist_search_key(DoublyLinkedList *list, void *key); int *dllist_search_range(DoublyLinkedList *list, void *start_key, bool include_start, void *end_key, bool include_end); diff --git a/src/commons/collections/include/pdc_pht.h b/src/commons/collections/include/pdc_pht.h index a4f139617..6ffa66412 100644 --- a/src/commons/collections/include/pdc_pht.h +++ b/src/commons/collections/include/pdc_pht.h @@ -1,4 +1,28 @@ +#ifndef PDC_PHT_H +#define PDC_PHT_H + +#include +#include +#include +#include +#include +#include "pdc_hash_table.h" +#include "pdc_utlist.h" + #define ROOT_PREFIX "#" + +typedef struct DoublyLinkedListItem { + char *key; + void *value; + struct DoublyLinkedList *prev; /* needed for a doubly-linked list only */ + struct DoublyLinkedList *next; /* needed for singly- or doubly-linked lists */ +} DoublyLinkedListItem; + +typedef struct DoublyLinkedList { + DoublyLinkedListItem *head; /* first element in list */ + unsigned int count; +} DoublyLinkedList; + typedef void* PrefixTableKey; typedef void* PrefixTableValue; @@ -9,7 +33,6 @@ typedef void* PrefixTableValue; * @param data The data to generate a hash value for. * @return The hash value. */ - typedef unsigned int (*PrefixTableHashFunc)(PrefixTableKey data); /** @@ -18,34 +41,50 @@ typedef unsigned int (*PrefixTableHashFunc)(PrefixTableKey data); * @return Non-zero if the two keys are equal, zero if the keys are * not equal. */ - typedef int (*PrefixTableEqualFunc)(PrefixTableKey value1, PrefixTableKey value2); /** * Type of function used to free keys when entries are removed from a * hash table. */ - typedef void (*PrefixTableKeyFreeFunc)(PrefixTableKey value); -/** - * Prefix table data structure - */ -typedef struct _PrefixTable PrefixTable; - /** * Prefix table bucket structure */ +struct _PrefixTableBucket { + bool isLeaf; // true if this is a leaf node + /** + * Parent and siblings + */ + char *parent; + char *left; + char *right; + + char *prefix; + DoublyLinkedList *store; +}; + +struct _PrefixTable { + HashTable *map; + size_t bucket_size; + size_t key_count; + PrefixTableHashFunc hash_cb; + PrefixTableEqualFunc equal_cb; +}; + typedef struct _PrefixTableBucket PrefixTableBucket; +typedef struct _PrefixTable PrefixTable; /** * Initializes a new prefix table. * * @param bucket_size The size of each bucket in the prefix table. - * @param free_cb Function to free keys when they are removed from the table. + * @param hash_cb Function to hash keys. + * @param equal_cb Function to compare keys for equality. * @return A pointer to the initialized prefix table, or NULL on failure. */ -PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb, PrefixTableKeyFreeFunc free_cb); +PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb); /** * Inserts a key-value pair into the prefix table. @@ -93,4 +132,23 @@ unsigned int prefix_table_count(PrefixTable *pht); * * @param pht The prefix table to destroy. */ -void prefix_table_destroy(PrefixTable *pht); \ No newline at end of file +void prefix_table_destroy(PrefixTable *pht); + +/** + * Initializes a new prefix table bucket. + * + * @param hash_cb Hash function callback. + * @param equal_cb Equality comparison callback. + * @return A pointer to the initialized bucket, or NULL on failure. + */ +PrefixTableBucket* prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb); + +/** + * Checks if a bucket is a leaf node. + * + * @param bucket The bucket to check. + * @return true if the bucket is a leaf, false otherwise. + */ +bool prefix_table_bucket_is_leaf(PrefixTableBucket *bucket); + +#endif /* PDC_PHT_H */ \ No newline at end of file diff --git a/src/server/include/pdc_utlist.h b/src/commons/collections/include/pdc_utlist.h similarity index 100% rename from src/server/include/pdc_utlist.h rename to src/commons/collections/include/pdc_utlist.h diff --git a/src/commons/collections/pdc_dllist.c b/src/commons/collections/pdc_dllist.c index 5cc4dc942..de51f4fa2 100644 --- a/src/commons/collections/pdc_dllist.c +++ b/src/commons/collections/pdc_dllist.c @@ -1,30 +1,27 @@ #include "pdc_dllist.h" -#include "pdc_set.h" #include -struct _DoublyLinkedListItem{ +// Now define the actual structures +struct DoublyLinkedListItem { void * key; void * value; - struct _DoublyLinkedListItem *prev; - struct _DoublyLinkedListItem *next; + struct DoublyLinkedListItem *prev; // Use 'struct' keyword + struct DoublyLinkedListItem *next; // Use 'struct' keyword }; -struct _DoublyLinkedList { +struct DoublyLinkedList { DoublyLinkedListItem * head; DoublyLinkedListHashFunc hash_func; DoublyLinkedListEqualFunc equal_func; - DoublyLinkedListKeyFreeFunc key_free_func; unsigned int count; }; - -DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func, DoublyLinkedListKeyFreeFunc key_free_func){ +DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func){ DoublyLinkedList *list = malloc(sizeof(DoublyLinkedList)); if (!list) { return NULL; // Memory allocation failed } list->hash_func = hash_func; list->equal_func = equal_func; - list->key_free_func = key_free_func; list->head = NULL; list->count = 0; return list; @@ -68,6 +65,8 @@ int dllist_destroy(void *ptr){ DoublyLinkedListItem *tmp = NULL; DL_FOREACH_SAFE(list->head, elt, tmp) { DL_DELETE(list->head, elt); + free(elt->value); + free(elt->key); free(elt); } } \ No newline at end of file diff --git a/src/commons/collections/pdc_pht.c b/src/commons/collections/pdc_pht.c index 7633dcb3b..ae594c87b 100644 --- a/src/commons/collections/pdc_pht.c +++ b/src/commons/collections/pdc_pht.c @@ -1,35 +1,13 @@ -#include -#include -#include -#include -#include -#include "pdc_hash_table.h" #include "pdc_pht.h" -#include "pdc_dllist.h" + #ifdef __cplusplus extern "C" { #endif -struct _PrefixTableBucket { - bool isLeaf; // true if this is a leaf node - /** - * Parent and siblings - */ - char *parent; - char *left; - char *right; - char *prefix; - DoublyLinkedList *store; -}; -struct _PrefixTable { - HashTable *map; - size_t bucket_size; - size_t key_count; -}; -PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb, PrefixTableKeyFreeFunc free_cb){ +PrefixTable* prefix_table_init(unsigned int bucket_size, HashTableHashFunc hash_cb, HashTableEqualFunc equal_cb){ PrefixTable *pht = (PrefixTable *)malloc(sizeof(PrefixTable)); if (!pht) { return NULL; // Memory allocation failed @@ -41,22 +19,32 @@ PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc has free(pht); return NULL; // Hash table creation failed } + pht->hash_cb = hash_cb; + pht->equal_cb = equal_cb; return pht; } -PrefixTableBucket* prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb, PrefixTableKeyFreeFunc free_cb){ +PrefixTableBucket* prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb){ PrefixTableBucket *bucket = (PrefixTableBucket *)malloc(sizeof(PrefixTableBucket)); if (!bucket) { return NULL; // Memory allocation failed } - bucket->isLeaf = false; + bucket->isLeaf = true; bucket->parent = NULL; bucket->left = NULL; bucket->right = NULL; bucket->prefix = NULL; - bucket->store = dllist_init(hash_cb, equal_cb, free_cb); + bucket->store = dllist_init(hash_cb, equal_cb); return bucket; } +int prefix_table_insert(PrefixTable *pht, void *key, void *value){ + if (!pht || !key || !value) { + return -1; // Invalid parameters + } + hash_table_insert(pht->map, key, value); + pht->key_count++; + return 0; +} void prefix_table_destroy(PrefixTable *pht){ hash_table_free(pht->map); @@ -66,7 +54,9 @@ void prefix_table_destroy(PrefixTable *pht){ void pht_add_to_store(PrefixTable *pht, void *key, void *value){ hash_table_insert(pht->map, key, value); } - +bool prefix_table_bucket_is_leaf(PrefixTableBucket *bucket){ + return bucket->isLeaf; +} #ifdef __cplusplus } diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index 9700d8628..76ad291c3 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -60,6 +60,7 @@ endif() add_library(pdc_server_lib pdc_server.c pdc_server_metadata_index.c + pdc_server_metadata_pht.c pdc_server_metadata.c pdc_client_server_common.c dablooms/pdc_dablooms.c diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index 0376c8523..f49eac643 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -65,6 +65,7 @@ extern struct timeval last_cache_activity_timeval_g; #define PDC_SEQ_ID_INIT_VALUE 1000 #define PDC_UPDATE_CACHE 111 #define PDC_UPDATE_STORAGE 101 +#define RING_ARC_SIZE 64 #ifndef HOST_NAME_MAX #if defined(__APPLE__) @@ -89,6 +90,8 @@ extern uint64_t pdc_id_seq_g; extern int pdc_server_rank_g; extern hg_atomic_int32_t close_server_g; hg_handle_t close_all_server_handle_g; +extern int pdc_server_size_g; +extern hg_id_t metadata_create_bucket_register_id_g; #define PDC_LOCK_OP_OBTAIN 0 #define PDC_LOCK_OP_RELEASE 1 @@ -523,8 +526,30 @@ typedef struct { typedef struct { int32_t ret; + bool found; + bool leaf; + uint32_t server_id; } metadata_check_prefix_out_t; +typedef struct { + char* prefix; +} metadata_create_bucket_in_t; + +typedef struct { + int32_t ret; +} metadata_create_bucket_out_t; + +typedef struct { + char* prefix; + char* key; + void* value; + uint32_t size; +} metadata_key_add_in_t; + +typedef struct { + int32_t ret; +} metadata_key_add_out_t; + /* Define region_lock_out_t */ typedef struct { int32_t ret; @@ -1738,12 +1763,75 @@ hg_proc_metadata_add_tag_in_t(hg_proc_t proc, void *data) static HG_INLINE hg_return_t hg_proc_metadata_check_prefix_in_t(hg_proc_t proc, void *data) { - return HG_SUCCESS; + FUNC_ENTER(NULL); + + hg_return_t ret; + metadata_check_prefix_in_t *struct_data = (metadata_check_prefix_in_t *)data; + + ret = hg_proc_hg_string_t(proc, &struct_data->prefix); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + + FUNC_LEAVE(ret); } static HG_INLINE hg_return_t hg_proc_metadata_check_prefix_out_t(hg_proc_t proc, void *data) { - return HG_SUCCESS; + FUNC_ENTER(NULL); + + hg_return_t ret; + metadata_check_prefix_out_t *struct_data = (metadata_check_prefix_out_t *)data; + + ret = hg_proc_hg_bool_t(proc, &struct_data->found); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + ret = hg_proc_hg_bool_t(proc, &struct_data->leaf); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + + ret = hg_proc_uint32_t(proc, &struct_data->server_id); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + ret = hg_proc_int32_t(proc, &struct_data->ret); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + FUNC_LEAVE(ret); +} + +static HG_INLINE hg_return_t +hg_proc_metadata_create_bucket_in_t(hg_proc_t proc, void *data) { + FUNC_ENTER(NULL); + + hg_return_t ret; + metadata_create_bucket_in_t *struct_data = (metadata_create_bucket_in_t *)data; + + printf("In hg_proc_metadata_create_bucket_in_t %s\n", struct_data->prefix); + ret = hg_proc_hg_string_t(proc, &struct_data->prefix); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + + FUNC_LEAVE(ret); +} + +static HG_INLINE hg_return_t +hg_proc_metadata_create_bucket_out_t(hg_proc_t proc, void *data) { + FUNC_ENTER(NULL); + + hg_return_t ret; + metadata_create_bucket_out_t *struct_data = (metadata_create_bucket_out_t *)data; + + ret = hg_proc_int32_t(proc, &struct_data->ret); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + + FUNC_LEAVE(ret); } /* Define hg_proc_metadata_get_kvtag_in_t */ @@ -1809,6 +1897,45 @@ hg_proc_metadata_get_kvtag_out_t(hg_proc_t proc, void *data) FUNC_LEAVE(ret); } +static HG_INLINE hg_return_t +hg_proc_metadata_key_add_in_t(hg_proc_t proc, void *data) +{ + FUNC_ENTER(NULL); + hg_return_t ret; + metadata_key_add_in_t *struct_data = (metadata_key_add_in_t *)data; + + ret = hg_proc_hg_string_t(proc, &struct_data->key); + if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + printf("In hg_proc_metadata_key_add_in_t Key:%s\n", struct_data->key); + + ret = hg_proc_hg_string_t(proc, &struct_data->prefix); + if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + printf("In hg_proc_metadata_key_add_in_t Prefix:%s\n", struct_data->prefix); + + ret = hg_proc_raw(proc, struct_data->value, struct_data->size); + if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + printf("In hg_proc_metadata_key_add_in_t Value:%p\n", struct_data->value); + + ret = hg_proc_uint32_t(proc, &struct_data->size); + if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + printf("In hg_proc_metadata_key_add_in_t Size:%d\n", struct_data->size); + + FUNC_LEAVE(ret); +} + +static HG_INLINE hg_return_t +hg_proc_metadata_key_add_out_t(hg_proc_t proc, void *data) +{ + FUNC_ENTER(NULL); + + hg_return_t ret; + metadata_key_add_out_t *struct_data = (metadata_key_add_out_t *)data; + + ret = hg_proc_int32_t(proc, &struct_data->ret); + if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + + FUNC_LEAVE(ret); +} static HG_INLINE hg_return_t hg_proc_metadata_add_kvtag_in_t(hg_proc_t proc, void *data) { @@ -4223,6 +4350,8 @@ hg_id_t PDC_metadata_update_register(hg_class_t *hg_class); hg_id_t PDC_metadata_add_tag_register(hg_class_t *hg_class); hg_id_t PDC_metadata_add_kvtag_register(hg_class_t *hg_class); hg_id_t PDC_metadata_check_prefix_register(hg_class_t *hg_class); +hg_id_t PDC_metadata_create_bucket_register(hg_class_t *hg_class); +hg_id_t PDC_metadata_key_add_register(hg_class_t *hg_class); hg_id_t PDC_metadata_del_kvtag_register(hg_class_t *hg_class); hg_id_t PDC_metadata_get_kvtag_register(hg_class_t *hg_class); hg_id_t PDC_send_rpc_register(hg_class_t *hg_class); @@ -4788,7 +4917,7 @@ char* string_to_binary(void *ptr); * * \return Hash value of the string */ -uint32_t prefix_hash(const char *key); +uint64_t prefix_hash(const char *key); /** * Duplicate a kvtag @@ -4856,4 +4985,6 @@ perr_t PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint6 struct pdc_region_info *region_info, void *buf, size_t unit, int is_write); +uint32_t PDC_get_server_using_pht(uint64_t key_hash); +perr_t PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out); #endif /* PDC_CLIENT_SERVER_COMMON_H */ diff --git a/src/server/include/pdc_server.h b/src/server/include/pdc_server.h index 2cc69d18e..7d80460a0 100644 --- a/src/server/include/pdc_server.h +++ b/src/server/include/pdc_server.h @@ -105,6 +105,7 @@ extern double server_update_time_g; extern double server_hash_insert_time_g; extern double server_bloom_init_time_g; extern uint32_t n_metadata_g; +extern int pdc_client_num_g; /***************************************/ /* Library-private Function Prototypes */ @@ -210,5 +211,6 @@ hg_return_t PDC_Server_recv_shm_cb(const struct hg_cb_info *callback_info); * \return Non-negative on success/Negative on failure */ int server_run(int argc, char *argv[]); - +perr_t PDC_Server2Server_create_bucket(char *prefix, uint32_t *server_id); +hg_return_t metadata_create_bucket_server_rpc_cb(const struct hg_cb_info *callback_info); #endif /* PDC_SERVER_H */ diff --git a/src/server/include/pdc_server_metadata.h b/src/server/include/pdc_server_metadata.h index 87291f90c..d853856fa 100644 --- a/src/server/include/pdc_server_metadata.h +++ b/src/server/include/pdc_server_metadata.h @@ -37,6 +37,7 @@ #include "pdc_server_common.h" #include "pdc_client_server_common.h" #include "pdc_server_metadata_index.h" +#include "pdc_server_metadata_pht.h" #include "pdc_malloc.h" @@ -380,14 +381,5 @@ perr_t PDC_free_cont_hash_table(); */ perr_t PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out); -/** - * Check if there is any object with the given prefix - * - * \param in [IN] Input structure received from client - * \param out [IN] Output structure to be sent back to the client - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out); #endif /* PDC_SERVER_METADATA_H */ diff --git a/src/server/include/pdc_server_metadata_pht.h b/src/server/include/pdc_server_metadata_pht.h index e69de29bb..315dbfac5 100644 --- a/src/server/include/pdc_server_metadata_pht.h +++ b/src/server/include/pdc_server_metadata_pht.h @@ -0,0 +1,29 @@ +#ifndef PDC_SERVER_METADATA_PHT_H +#define PDC_SERVER_METADATA_PHT_H + +#include "pdc_client_server_common.h" + +#include "query_utils.h" +#include "timer_utils.h" +#include "pdc_set.h" +#include "pdc_hash.h" +#include "pdc_compare.h" +#include "pdc_hash_table.h" +#include "pdc_pht.h" + +#define PHT_BUCKET_SIZE 1024 + +void PDC_Server_metadata_pht_init(uint32_t num_server, uint32_t server_id); +perr_t PDC_Server_add_metadata(); +/** + * Create a bucket with the given prefix + * + * \param in [IN] Input structure received from client + * \param out [IN] Output structure to be sent back to the client + * + * \return Non-negative on success/Negative on failure + */ +perr_t PDC_Server_create_bucket(metadata_create_bucket_in_t *in, metadata_create_bucket_out_t *out); +perr_t PDC_Server_metadata_key_add(metadata_key_add_in_t *in, metadata_key_add_out_t *out); + +#endif /* PDC_SERVER_METADATA_PHT_H */ \ No newline at end of file diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index d4c534e4d..8d058c343 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -55,6 +55,11 @@ #include "pdc_timing.h" #include "pdc_server_region_cache.h" +int pdc_server_size_g = 1; +static hg_context_t * send_context_g = NULL; +hg_id_t metadata_create_bucket_register_id_g; +static hg_atomic_int32_t atomic_work_todo_g; + #ifdef ENABLE_MULTITHREAD hg_thread_mutex_t insert_metadata_mutex_g = HG_THREAD_MUTEX_INITIALIZER; @@ -224,10 +229,11 @@ PDC_get_server_by_obj_id(uint64_t obj_id, int n_server) FUNC_LEAVE(ret_value); } -uint32_t prefix_hash(const char *key){ - uint32_t hash = 0; +uint64_t prefix_hash(const char *key){ + uint64_t hash = 0; + uint64_t len = strlen(key); int i = 1; - while (*key) { + while (i < len) { if (*key == '\0') break; if (*key == '#') { *key++; @@ -261,7 +267,7 @@ char* string_to_binary(void *ptr){ uint32_t PDC_get_server_by_obj_id_pht(uint64_t obj_id, int n_server) { - + return SUCCEED; } int @@ -1012,6 +1018,19 @@ PDC_Server_check_prefix(metadata_check_prefix_in_t *in ATTRIBUTE(unused), { return SUCCEED; } + +perr_t +PDC_Server_create_bucket(metadata_create_bucket_in_t *in ATTRIBUTE(unused), + metadata_create_bucket_out_t *out ATTRIBUTE(unused)){ + return SUCCEED; +} + +perr_t +PDC_Server_metadata_key_add(metadata_key_add_in_t *in ATTRIBUTE(unused), + metadata_key_add_out_t *out ATTRIBUTE(unused)) +{ + return SUCCEED; +} perr_t PDC_Meta_Server_buf_unmap(buf_unmap_in_t *in ATTRIBUTE(unused), hg_handle_t *handle ATTRIBUTE(unused)) { @@ -1916,6 +1935,57 @@ HG_TEST_RPC_CB(metadata_check_prefix, handle) FUNC_LEAVE(ret_value); } +HG_TEST_RPC_CB(metadata_key_add, handle) +{ + FUNC_ENTER(NULL); + + hg_return_t ret_value = HG_SUCCESS; + metadata_key_add_in_t in; + metadata_key_add_out_t out; + + printf("metadata_key_add %s\n", &in.prefix); + HG_Get_input(handle, &in); + if (strcmp(in.prefix, "PDC_NOOP") != 0) { + printf("PDC_Server_metadata_key_add\n"); + PDC_Server_metadata_key_add(&in, &out); + } else { + LOG_INFO("Received NOOP\n"); + out.ret = 1; + } + + ret_value = HG_Respond(handle, NULL, NULL, &out); + + HG_Free_input(handle, &in); + HG_Destroy(handle); + + FUNC_LEAVE(ret_value); +} + +HG_TEST_RPC_CB(metadata_create_bucket, handle) +{ + FUNC_ENTER(NULL); + + hg_return_t ret_value = HG_SUCCESS; + metadata_create_bucket_in_t in; + metadata_create_bucket_out_t out; + + HG_Get_input(handle, &in); + if (strcmp(in.prefix, "PDC_NOOP") != 0) { + PDC_Server_create_bucket(&in, &out); + } + else { + LOG_INFO("Received NOOP\n"); + out.ret = 1; + } + + ret_value = HG_Respond(handle, NULL, NULL, &out); + + HG_Free_input(handle, &in); + HG_Destroy(handle); + + FUNC_LEAVE(ret_value); +} + /* static hg_return_t */ // notify_io_complete_cb(hg_handle_t handle) HG_TEST_RPC_CB(notify_io_complete, handle) @@ -6233,6 +6303,8 @@ PDC_FUNC_DECLARE_REGISTER(send_rpc) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_del_kvtag, metadata_get_kvtag_in_t, metadata_add_tag_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_add_kvtag, metadata_add_kvtag_in_t, metadata_add_tag_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_check_prefix, metadata_check_prefix_in_t, metadata_check_prefix_out_t) +PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_key_add, metadata_key_add_in_t, metadata_key_add_out_t) +PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_create_bucket, metadata_create_bucket_in_t, metadata_create_bucket_out_t) PDC_FUNC_DECLARE_REGISTER(metadata_get_kvtag) PDC_FUNC_DECLARE_REGISTER(metadata_update) PDC_FUNC_DECLARE_REGISTER(metadata_delete_by_id) @@ -6995,3 +7067,20 @@ PDCselection_print_all(pdc_selection_t *sel) FUNC_LEAVE_VOID(); } + +uint32_t +PDC_get_server_using_pht(uint64_t key_hash) { + int ring_size = pdc_server_size_g * RING_ARC_SIZE; + uint64_t mapped_hash = key_hash % ring_size; + + for (int i = 0; i < pdc_server_size_g; i++) { + int node_pos = (i + 1) * ceil(ring_size / (float)pdc_server_size_g); + + if (mapped_hash <= node_pos) { + return i; + } + } + + // Wrap around + return 0; +} \ No newline at end of file diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index b8b405d84..a31c6f631 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -128,7 +128,6 @@ hg_atomic_int32_t close_server_g; char pdc_server_tmp_dir_g[TMP_DIR_STRING_LEN]; int is_restart_g = 0; int pdc_server_rank_g = 0; -int pdc_server_size_g = 1; int write_to_bb_percentage_g = 0; int pdc_nost_per_file_g = 0; int nclient_per_node = 0; @@ -147,7 +146,6 @@ int gen_fastbit_idx_g = 0; int use_fastbit_idx_g = 0; int use_rocksdb_g = 0; int use_sqlite3_g = 0; -int use_pht_g = 0; char * gBinningOption = NULL; double server_write_time_g = 0.0; @@ -923,6 +921,8 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) LOG_INFO("Read cache enabled\n"); #endif + // Initialize PHT for metadata + PDC_Server_metadata_pht_init(pdc_server_size_g, pdc_server_rank_g); // Initialize IDIOMS PDC_Server_metadata_index_init(pdc_server_size_g, pdc_server_rank_g); @@ -1930,6 +1930,8 @@ PDC_Server_mercury_register() PDC_gen_cont_id_register(hg_class_g); PDC_metadata_add_kvtag_register(hg_class_g); PDC_metadata_check_prefix_register(hg_class_g); + PDC_metadata_create_bucket_register(hg_class_g); + PDC_metadata_key_add_register(hg_class_g); PDC_metadata_get_kvtag_register(hg_class_g); PDC_metadata_del_kvtag_register(hg_class_g); PDC_send_rpc_register(hg_class_g); @@ -2123,6 +2125,71 @@ PDC_Server_get_env() FUNC_LEAVE_VOID(); } +perr_t +PDC_Server2Server_create_bucket(char *prefix, uint32_t *server_id) { + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + hg_handle_t metadata_create_bucket_handle; + metadata_create_bucket_in_t in; + metadata_create_bucket_out_t out; + + uint64_t hash_value = prefix_hash(prefix); + *server_id = PDC_get_server_using_pht(hash_value); + + in.prefix = strdup(prefix); + if (*server_id == pdc_server_rank_g){ + ret_value = PDC_Server_create_bucket(&in, &out); + if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Failed to get local storage location"); + } else { + if (PDC_Server_lookup_server_id(*server_id) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); + + hg_ret = HG_Create(hg_context_g, pdc_remote_server_info_g[*(uint32_t *)server_id].addr, metadata_create_bucket_register_id_g, + &metadata_create_bucket_handle); + + hg_ret = HG_Forward(metadata_create_bucket_handle, metadata_create_bucket_server_rpc_cb, NULL, &in); + + if (hg_ret != HG_SUCCESS){ + HG_Destroy(metadata_create_bucket_handle); + PGOTO_ERROR(FAIL, "Could not start HG_Forward"); + } + + if (hg_ret != SUCCEED) + LOG_ERROR("Add create_bucket server2server NOT successful"); + + HG_Destroy(metadata_create_bucket_handle); + } +done: + FUNC_LEAVE(ret_value); +} + +hg_return_t +metadata_create_bucket_server_rpc_cb(const struct hg_cb_info *callback_info) +{ + + FUNC_ENTER(NULL); + hg_return_t ret_value; + hg_handle_t handle = callback_info->info.forward.handle; + metadata_create_bucket_out_t *result = (metadata_create_bucket_out_t *)callback_info->arg; + /* Get output from server*/ + + metadata_create_bucket_out_t output; + ret_value = HG_Get_output(handle, &output); + result = &output; + + if (ret_value != HG_SUCCESS) { + PGOTO_ERROR(HG_OTHER_ERROR, "Error with HG_Get_output"); + } + printf("metadata_create_bucket_rpc_cb: output.ret = %d\n", output.ret); +done: + + // hg_atomic_decr32(&atomic_work_todo_g); + HG_Free_output(handle, &output); + FUNC_LEAVE(ret_value); +} + int server_run(int argc, char *argv[]) { diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index 05ef6d5dd..ec8fde0bd 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -41,6 +41,7 @@ #include "mpi.h" #endif +#include "pdc_pht.h" #include "pdc_utlist.h" #include "pdc_hash_table.h" #include "pdc_dablooms.h" @@ -449,7 +450,6 @@ PDC_Server_init_hash_table() PDC_Server_container_hash_value_free); is_hash_table_init_g = 1; - done: FUNC_LEAVE(ret_value); } @@ -2753,7 +2753,7 @@ PDC_Server_add_kvtag_someta(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t perr_t PDC_Server_add_metadata_key(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) { - + return SUCCEED; } perr_t PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) @@ -2828,66 +2828,6 @@ PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out) FUNC_LEAVE(ret_value); } -perr_t -PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out) -{ - perr_t ret_value = SUCCEED; -#ifdef ENABLE_MULTITHREAD - int unlocked; -#endif - FUNC_ENTER(NULL); - -#ifdef ENABLE_TIMING - struct timeval pdc_timer_start; - struct timeval pdc_timer_end; - double ht_total_sec; - gettimeofday(&pdc_timer_start, 0); -#endif - - out->ret = -1; - -#ifdef ENABLE_MULTITHREAD - // Obtain lock for hash table - unlocked = 0; - hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); -#endif - - out->ret = 0; - -done: -#ifdef ENABLE_MULTITHREAD - // ^ Release hash table lock - hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); - unlocked = 1; -#endif - -#ifdef ENABLE_TIMING - // Timing - gettimeofday(&pdc_timer_end, 0); - ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); -#endif - -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&pdc_time_mutex_g); -#endif - -#ifdef ENABLE_TIMING - server_update_time_g += ht_total_sec; -#endif - -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&pdc_time_mutex_g); -#endif - -#ifdef ENABLE_MULTITHREAD - if (unlocked == 0) - hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); -#endif - fflush(stdout); - - FUNC_LEAVE(ret_value); -} - static perr_t PDC_get_kvtag_value_from_list(pdc_kvtag_list_t **list_head, char *key, metadata_get_kvtag_out_t *out) { diff --git a/src/server/pdc_server_metadata_pht.c b/src/server/pdc_server_metadata_pht.c index 87b3e9385..be5bbdbf1 100644 --- a/src/server/pdc_server_metadata_pht.c +++ b/src/server/pdc_server_metadata_pht.c @@ -55,108 +55,366 @@ #include "pdc_pht.h" -PrefixTable *metadata_pht_g = NULL; +uint32_t metadata_server_id_g = 0; +uint32_t metadata_num_server_g = 0; -/* - * Check if two hash keys are equal - * - * \param vlocation1 [IN] Hash table key - * \param vlocation2 [IN] Hash table key - * - * \return 1 if two keys are equal, 0 otherwise - */ -static int -PDC_Server_metadata_pht_int_compare(void *vlocation1, void *vlocation2) -{ - int32_t k1 = *(int32_t *) key1; - int32_t k2 = *(int32_t *) key2; +PrefixTable *metadata_pht_key_g = NULL; + +unsigned int pht_string_hash(void *vlocation) { + unsigned int result = 5381; + unsigned char *p; + + p = (unsigned char *)vlocation; + + while (*p != '\0') { + result = (result << 5) + result + *p; + ++p; + } - return (k1 > k2) - (k1 < k2); + return result; } -/* - * Get hash key's location in hash table - * - * \param vlocation [IN] Hash table key - * - * \return the location of hash key in the table - */ -static unsigned int -PDC_Server_metadata_pht_int_hash(void *vlocation) -{ - int *location; - location = (int *) vlocation; - return (unsigned int) *location; +int pht_string_comparator(const void *key1, const void *key2){ + char *ch1 = (char *) key1; + char *ch2 = (char *) key2; + int diff = strcmp(ch1, ch2); + return diff < 0 ? -1 : diff > 0 ? 1 : 0; } -/* - * Free the hash key - * - * \param key [IN] Hash table key - * - * \return void - */ -static void -PDC_Server_metadata_pht_int_hash_key_free(void *key) +/****************************/ +/* Initialize PHT */ +/****************************/ +void +PDC_Server_metadata_pht_init(uint32_t num_server, uint32_t server_id) { - free(key); + FUNC_ENTER(NULL); + + metadata_server_id_g = num_server; + metadata_num_server_g = server_id; + metadata_pht_key_g = prefix_table_init(PHT_BUCKET_SIZE, string_hash, string_equal); + + FUNC_LEAVE_VOID(); } -/* - * Free metadata hash value - * - * \param value [IN] Hash table value - * - * \return void - */ -static void -PDC_Server_metadata_pht_value_free(void *value) +perr_t +PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out) { - pdc_metadata_t *elt, *tmp; - PrefixTableBucket *head; + perr_t ret_value = SUCCEED; +#ifdef ENABLE_MULTITHREAD + int unlocked; +#endif + FUNC_ENTER(NULL); + +#ifdef ENABLE_TIMING + struct timeval pdc_timer_start; + struct timeval pdc_timer_end; + double ht_total_sec; + gettimeofday(&pdc_timer_start, 0); +#endif + +#ifdef ENABLE_MULTITHREAD + // Obtain lock for hash table + unlocked = 0; + hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); +#endif + printf("PDC_Server_check_prefix: in->prefix = %s\n", in->prefix); + printf("PDC_Server_check_prefix: metadata_pht_key_g = %d\n", metadata_pht_key_g->key_count); + HashTableValue *value = hash_table_lookup(metadata_pht_key_g->map, in->prefix); + printf("PDC_Server_check_prefix: value = %p\n", value); + PrefixTableBucket *bucket = (PrefixTableBucket *) value; + printf("PDC_Server_check_prefix: isLeaf = %d\n", bucket->isLeaf); + out->found = (bucket != NULL) ? 1 : 0; + out->leaf = prefix_table_bucket_is_leaf(bucket); + out->ret = 1; + out->server_id = metadata_server_id_g; + +done: +#ifdef ENABLE_MULTITHREAD + // ^ Release hash table lock + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); + unlocked = 1; +#endif + +#ifdef ENABLE_TIMING + // Timing + gettimeofday(&pdc_timer_end, 0); + ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); +#endif +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_TIMING + server_update_time_g += ht_total_sec; +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_unlock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_MULTITHREAD + if (unlocked == 0) + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); +#endif + fflush(stdout); + + FUNC_LEAVE(ret_value); +} + +perr_t +PDC_Server_create_bucket(metadata_create_bucket_in_t *in, metadata_create_bucket_out_t *out) +{ + perr_t ret_value = SUCCEED; +#ifdef ENABLE_MULTITHREAD + int unlocked; +#endif FUNC_ENTER(NULL); - head = (PrefixTableBucket *)value; +#ifdef ENABLE_TIMING + struct timeval pdc_timer_start; + struct timeval pdc_timer_end; + double ht_total_sec; + gettimeofday(&pdc_timer_start, 0); +#endif - // Free metadata list - if (is_restart_g == 0) { - DL_FOREACH_SAFE(head.store, elt, tmp) - { - free(elt); - } +#ifdef ENABLE_MULTITHREAD + // Obtain lock for hash table + unlocked = 0; + hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); +#endif + if (metadata_pht_key_g == NULL) { + perror("metadata_pht_key_g is NULL"); + ret_value = FAIL; + goto done; + } + printf("PDC_Server_create_bucket: in->prefix = %s\n", in->prefix); + PrefixTableBucket *bucket = hash_table_lookup(metadata_pht_key_g->map, in->prefix); + if (bucket != NULL) { + out->ret = 1; + goto done; // already exists } + PrefixTableBucket *new_bucket = prefix_table_bucket_init(pht_string_hash, pht_string_comparator); + new_bucket->prefix = strdup(in->prefix); + hash_table_insert(metadata_pht_key_g->map, new_bucket->prefix, new_bucket); + printf("PDC_Server_create_bucket: new_bucket = %p\n", new_bucket); + out->ret = 1; + if (bucket == NULL) { + ret_value = FAIL; + goto done; + } +done: +#ifdef ENABLE_MULTITHREAD + // ^ Release hash table lock + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); + unlocked = 1; +#endif + +#ifdef ENABLE_TIMING + // Timing + gettimeofday(&pdc_timer_end, 0); + ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_TIMING + server_update_time_g += ht_total_sec; +#endif + +#ifdef ENABLE_MULTITHREAD + if (unlocked == 0) + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); +#endif + fflush(stdout); + + FUNC_LEAVE(ret_value); } -/* - * Get the metadata with obj ID from the metadata list - * - * \param mlist[IN] Metadata list head - * \param obj_id[IN] Object ID - * - * \return NULL if no match is found/pointer to the found metadata otherwise - */ -static pdc_metadata_t * -find_metadata_pht_by_id_from_list(pdc_metadata_t *mlist, uint64_t obj_id) -{ - pdc_metadata_t *ret_value, *elt; +perr_t +PDC_Server_bucket_split(metadata_create_bucket_in_t *in, metadata_create_bucket_out_t *out) +{ + perr_t ret_value = SUCCEED; +#ifdef ENABLE_MULTITHREAD + int unlocked; +#endif FUNC_ENTER(NULL); - ret_value = NULL; - if (mlist == NULL) { - ret_value = NULL; +#ifdef ENABLE_TIMING + struct timeval pdc_timer_start; + struct timeval pdc_timer_end; + double ht_total_sec; + gettimeofday(&pdc_timer_start, 0); +#endif + +#ifdef ENABLE_MULTITHREAD + // Obtain lock for hash table + unlocked = 0; + hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); +#endif + if (metadata_pht_key_g == NULL) { + perror("metadata_pht_key_g is NULL"); + ret_value = FAIL; goto done; } - - DL_FOREACH(mlist, elt) - { - if (elt->obj_id == obj_id) { - ret_value = elt; - goto done; - } + printf("PDC_Server_create_bucket: in->prefix = %s\n", in->prefix); + PrefixTableBucket *bucket = hash_table_lookup(metadata_pht_key_g->map, in->prefix); + if (bucket != NULL) { + out->ret = 1; + goto done; // already exists } + PrefixTableBucket *new_bucket = prefix_table_bucket_init(pht_string_hash, pht_string_comparator); + new_bucket->prefix = strdup(in->prefix); + hash_table_insert(metadata_pht_key_g->map, new_bucket->prefix, new_bucket); + printf("PDC_Server_create_bucket: new_bucket = %p\n", new_bucket); + out->ret = 1; + if (bucket == NULL) { + ret_value = FAIL; + goto done; + } +done: +#ifdef ENABLE_MULTITHREAD + // ^ Release hash table lock + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); + unlocked = 1; +#endif + +#ifdef ENABLE_TIMING + // Timing + gettimeofday(&pdc_timer_end, 0); + ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_TIMING + server_update_time_g += ht_total_sec; +#endif + +#ifdef ENABLE_MULTITHREAD + if (unlocked == 0) + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); +#endif + fflush(stdout); + + FUNC_LEAVE(ret_value); +} + +perr_t +PDC_Server_metadata_key_add(metadata_key_add_in_t *in, metadata_key_add_out_t *out) +{ + perr_t ret_value = SUCCEED; +#ifdef ENABLE_MULTITHREAD + int unlocked; +#endif + FUNC_ENTER(NULL); + +#ifdef ENABLE_TIMING + struct timeval pdc_timer_start; + struct timeval pdc_timer_end; + double ht_total_sec; + gettimeofday(&pdc_timer_start, 0); +#endif +#ifdef ENABLE_MULTITHREAD + // Obtain lock for hash table + unlocked = 0; + hg_thread_mutex_lock(&pdc_metadata_hash_table_mutex_g); +#endif + printf("PDC_Server_metadata_key_add: in->prefix = %s\n", in->prefix); + printf("PDC_Server_metadata_key_add: metadata_pht_key_g = %d\n", metadata_pht_key_g->key_count); + HashTableValue *value = hash_table_lookup(metadata_pht_key_g->map, in->prefix); + printf("PDC_Server_metadata_key_add: value = %p\n", value); + PrefixTableBucket *bucket = (PrefixTableBucket *) value; + if (bucket == NULL) { + perror("bucket is NULL"); + ret_value = FAIL; + goto done; + } + printf("PDC_Server_metadata_key_add: key = %s\n", in->key); + Set *prevSet = dllist_search_key(bucket->store, in->key); + if (prevSet != NULL) { + set_insert(prevSet, in->value); + } else { + if (bucket->store->count >= PHT_BUCKET_SIZE) pht_bucket_split(bucket); + Set *newSet = set_new(metadata_pht_key_g->hash_cb, metadata_pht_key_g->equal_cb); + set_insert(newSet, in->value); + dllist_insert(bucket->store, strdup(in->key), newSet); + } done: +#ifdef ENABLE_MULTITHREAD + // ^ Release hash table lock + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); + unlocked = 1; +#endif + +#ifdef ENABLE_TIMING + // Timing + gettimeofday(&pdc_timer_end, 0); + ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_TIMING + server_update_time_g += ht_total_sec; +#endif + +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_unlock(&pdc_time_mutex_g); +#endif + +#ifdef ENABLE_MULTITHREAD + if (unlocked == 0) + hg_thread_mutex_unlock(&pdc_metadata_hash_table_mutex_g); +#endif + fflush(stdout); + FUNC_LEAVE(ret_value); +} + +int pht_bucket_split(PrefixTableBucket *bucket){ + char *left_prefix = malloc(sizeof(char) * (strlen(bucket->prefix) + 2)); + char *right_prefix = malloc(sizeof(char) * (strlen(bucket->prefix) + 2)); + sprintf(left_prefix, "%s0", bucket->prefix); + sprintf(right_prefix, "%s1", bucket->prefix); + uint32_t left_server_id, right_server_id; + PDC_Server2Server_create_bucket(left_prefix, &left_server_id); + PDC_Server2Server_create_bucket(right_prefix, &right_server_id); + bucket->isLeaf = false; + bucket->left = left_prefix; + bucket->right = right_prefix; + + DoublyLinkedListItem *elt = NULL; + DL_FOREACH(bucket->store->head, elt) { + char key_prefix = string_to_binary(elt->key); + bool is_left = strcmp(key_prefix, left_prefix) == 0; + bool is_right = strcmp(key_prefix, right_prefix) == 0; + uint32_t server_id = is_left ? left_server_id : right_server_id; + if (is_left || is_right){ + if (server_id == metadata_server_id_g) { + PrefixTableBucket *child_bucket = hash_table_lookup(metadata_pht_key_g->map, is_left ? left_prefix : right_prefix); + if (child_bucket == NULL) return -1; // error + dllist_insert(child_bucket->store, elt->key, elt->value); + } else { + metadata_key_add_in_t add_in; + metadata_key_add_out_t add_out; + add_in.prefix = is_left ? left_prefix : right_prefix; + add_in.key = elt->key; + add_in.value = elt->value; + add_in.size = sizeof(Set *); + //PDC_Client_metadata_key_add(server_id, &add_in, &add_out); + } + } + } + //dllist_destroy(bucket->store); + bucket->store = NULL; + return 0; } \ No newline at end of file diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index bf9935c84..86db205f8 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -84,6 +84,8 @@ set(PROGRAMS deprecated/bdcats_old tags/kvtag_add_get tags/kvtag_query + pht/pht_create_bucket + pht/pht_add_key region/region_transfer_query region/region_transfer region/region_transfer_status @@ -260,8 +262,10 @@ add_test(NAME obj_life WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTO add_test(NAME obj_buf WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_buf ) add_test(NAME obj_tags WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_tags ) add_test(NAME kvtag_add_get WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./kvtag_add_get) -add_test(NAME kvtag_query WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./kvtag_query 100 1 10 0) -add_test(NAME idioms_ci_test WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./idioms_ci_test) +add_test(NAME kvtag_query WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./kvtag_query 100 1 10 0) +add_test(NAME pht_create_bucket WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./pht_create_bucket) +add_test(NAME pht_add_key WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./pht_add_key) +add_test(NAME idioms_ci_test WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./idioms_ci_test) add_test(NAME obj_info WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_info ) add_test(NAME obj_put_data WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_put_data ) add_test(NAME obj_get_data WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./obj_get_data ) diff --git a/src/tests/pht/pht_add_key.c b/src/tests/pht/pht_add_key.c new file mode 100644 index 000000000..9b6a18bf5 --- /dev/null +++ b/src/tests/pht/pht_add_key.c @@ -0,0 +1,89 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +int +main() +{ + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t kvtag1, kvtag2, kvtag3; + char * v1 = "value1"; + int v2 = 2; + double v3 = 3.45; + pdc_var_type_t type1, type2, type3; + void * value1, *value2, *value3; + psize_t value_size; + int ret_value = SUCCEED; + int rank = 0; + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + TASSERT((cont = PDCcont_create("c1", cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create object properties + TASSERT((obj_prop1 = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + TASSERT((obj_prop2 = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create first object + TASSERT((obj1 = PDCobj_create(cont, "o1", obj_prop1)) != 0, "Call to PDCobj_create succeeded for obj1", + "Call to PDCobj_create failed for obj1"); + // create second object + TASSERT((obj2 = PDCobj_create(cont, "o2", obj_prop2)) != 0, "Call to PDCobj_create succeeded for obj2", + "Call to PDCobj_create failed for obj2"); + + kvtag1.name = "key1string"; + kvtag1.value = (void *)v1; + kvtag1.type = PDC_STRING; + kvtag1.size = strlen(v1) + 1; + + kvtag2.name = "key2int"; + kvtag2.value = (void *)&v2; + kvtag2.type = PDC_INT; + kvtag2.size = sizeof(int); + + kvtag3.name = "key3double"; + kvtag3.value = (void *)&v3; + kvtag3.type = PDC_DOUBLE; + kvtag3.size = sizeof(double); + + TASSERT(PDC_metadata_key_add(obj1, &kvtag1, false) >= 0, + "Call to PDC_metadata_key_add succeeded for obj1", "Call to PDC_metadata_key_add failed for obj1"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + +done: + return ret_value; +} diff --git a/src/tests/pht/pht_create_bucket.c b/src/tests/pht/pht_create_bucket.c new file mode 100644 index 000000000..674a478f8 --- /dev/null +++ b/src/tests/pht/pht_create_bucket.c @@ -0,0 +1,55 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +int +main() +{ + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t kvtag1, kvtag2, kvtag3; + char * v1 = "value1"; + int v2 = 2; + double v3 = 3.45; + psize_t value_size; + int ret_value = SUCCEED; + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + + TASSERT(PDC_Client_create_bucket("#") == 0, "Call to PDC_create_bucket succeeded", + "Call to PDC_create_bucket failed"); + + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + +done: + return ret_value; +} From c64a407a64a0d314cd361c658019167bc0c44467 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 6 Oct 2025 14:50:48 +0000 Subject: [PATCH 3/3] Committing clang-format changes --- src/api/include/pdc_client_connect.h | 10 +- src/api/pdc_client_connect.c | 148 ++++++++++-------- src/commons/collections/include/pdc_dllist.h | 18 +-- src/commons/collections/include/pdc_pht.h | 30 ++-- src/commons/collections/include/pdc_set.h | 3 +- src/commons/collections/pdc_dllist.c | 73 +++++---- src/commons/collections/pdc_pht.c | 39 +++-- src/commons/collections/pdc_set.c | 4 +- src/server/include/pdc_client_server_common.h | 71 +++++---- src/server/include/pdc_server.h | 4 +- src/server/include/pdc_server_metadata.h | 1 - src/server/include/pdc_server_metadata_pht.h | 2 +- src/server/pdc_client_server_common.c | 93 ++++++----- src/server/pdc_server.c | 47 +++--- src/server/pdc_server_metadata_pht.c | 86 +++++----- src/tests/pht/pht_add_key.c | 5 +- src/tests/pht/pht_create_bucket.c | 14 +- 17 files changed, 361 insertions(+), 287 deletions(-) diff --git a/src/api/include/pdc_client_connect.h b/src/api/include/pdc_client_connect.h index 72e7ea9f9..03d8ce6fc 100644 --- a/src/api/include/pdc_client_connect.h +++ b/src/api/include/pdc_client_connect.h @@ -1141,10 +1141,10 @@ perr_t PDC_Client_transfer_pthread_create(); perr_t PDC_Client_transfer_pthread_terminate(); perr_t PDC_Client_transfer_pthread_cnt_add(int n); -perr_t PDC_Client_create_bucket(char *prefix); -uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix); -perr_t PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); -perr_t PDC_metadata_key_delete(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); -perr_t PDC_delete_metadata_key(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); +perr_t PDC_Client_create_bucket(char *prefix); +uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix); +perr_t PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); +perr_t PDC_metadata_key_delete(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); +perr_t PDC_delete_metadata_key(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont); hg_return_t metadata_create_bucket_client_rpc_cb(const struct hg_cb_info *callback_info); #endif /* PDC_CLIENT_CONNECT_H */ diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index d63ec6dc9..59e42f25b 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -2055,10 +2055,10 @@ metadata_check_prefix_rpc_cb(const struct hg_cb_info *callback_info) { FUNC_ENTER(NULL); - hg_return_t ret_value; - metadata_check_prefix_out_t output; - metadata_check_prefix_out_t *args = (metadata_check_prefix_out_t *)callback_info->arg; - hg_handle_t handle = callback_info->info.forward.handle; + hg_return_t ret_value; + metadata_check_prefix_out_t output; + metadata_check_prefix_out_t *args = (metadata_check_prefix_out_t *)callback_info->arg; + hg_handle_t handle = callback_info->info.forward.handle; printf("metadata_check_prefix_rpc_cb: callback_info->arg = %p\n", callback_info->arg); /* Get output from server*/ @@ -2068,10 +2068,10 @@ metadata_check_prefix_rpc_cb(const struct hg_cb_info *callback_info) ret_value = -1; PGOTO_ERROR(HG_OTHER_ERROR, "Error with HG_Get_output"); } - args->found = output.found; + args->found = output.found; args->server_id = output.server_id; - args->ret = output.ret; - args->leaf = output.leaf; + args->ret = output.ret; + args->leaf = output.leaf; printf("metadata_check_prefix_rpc_cb: output.found = %d\n", args->found); printf("metadata_check_prefix_rpc_cb: output.leaf = %d\n", args->leaf); printf("metadata_check_prefix_rpc_cb: output.ret = %d\n", args->ret); @@ -2089,10 +2089,10 @@ metadata_key_add_rpc_cb(const struct hg_cb_info *callback_info) { FUNC_ENTER(NULL); - hg_return_t ret_value; - metadata_key_add_out_t output; - metadata_key_add_out_t *args = (metadata_key_add_out_t *)callback_info->arg; - hg_handle_t handle = callback_info->info.forward.handle; + hg_return_t ret_value; + metadata_key_add_out_t output; + metadata_key_add_out_t *args = (metadata_key_add_out_t *)callback_info->arg; + hg_handle_t handle = callback_info->info.forward.handle; printf("metadata_key_add_rpc_cb: callback_info->arg = %p\n", callback_info->arg); /* Get output from server*/ @@ -6471,12 +6471,13 @@ PDC_add_kvtag(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) * Check if the prefix exists in the PHT * @param server: The server to check against * @param prefix: The prefix to check - * + * * This function will send mercury RPC request to the server and check if the prefix exists */ static perr_t -PDC_check_prefix(uint32_t server, char *prefix, metadata_check_prefix_out_t *out) { +PDC_check_prefix(uint32_t server, char *prefix, metadata_check_prefix_out_t *out) +{ FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; hg_return_t hg_ret = 0; @@ -6510,38 +6511,42 @@ PDC_check_prefix(uint32_t server, char *prefix, metadata_check_prefix_out_t *out FUNC_LEAVE(ret_value); } -uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix){ +uint32_t +PDC_prefix_binary_search(char *prefix, char **target_prefix) +{ metadata_check_prefix_out_t resp; - uint64_t temp_server_id = -1; - int max = strlen(prefix) - 1; - int mid = 1; - char *slice; - + uint64_t temp_server_id = -1; + int max = strlen(prefix) - 1; + int mid = 1; + char * slice; + printf("Searching prefix %s\n", prefix); - while (mid <= max){ + while (mid <= max) { slice = malloc((mid + 1) * sizeof(char)); strncpy(slice, prefix, mid); - slice[mid] = '\0'; + slice[mid] = '\0'; uint64_t hash_value = prefix_hash(prefix); - temp_server_id = PDC_get_server_using_pht(hash_value); + temp_server_id = PDC_get_server_using_pht(hash_value); printf("Server %d is responsible for prefix %s till length %d\n", temp_server_id, slice, mid); perr_t ret = PDC_check_prefix(temp_server_id, slice, &resp); - - printf("Checking prefix %s on server %d: found=%d, leaf=%d\n", slice, temp_server_id, resp.found, resp.leaf); - //TODO: check if redirect to another server + + printf("Checking prefix %s on server %d: found=%d, leaf=%d\n", slice, temp_server_id, resp.found, + resp.leaf); + // TODO: check if redirect to another server /*** If the prefix exists and a leaf node, return the target server */ - if (resp.found == 1 && resp.leaf == 1){ + if (resp.found == 1 && resp.leaf == 1) { *target_prefix = malloc((mid + 1) * sizeof(char)); strncpy(*target_prefix, slice, mid); (*target_prefix)[mid] = '\0'; free(slice); return temp_server_id; } - else if (resp.found == 1 && resp.leaf == 0){ /** If the prefix exists but not a leaf node, search longer prefix */ + else if (resp.found == 1 && + resp.leaf == 0) { /** If the prefix exists but not a leaf node, search longer prefix */ mid++; } - else{ /** If the prefix does not exist, search shorter prefix */ + else { /** If the prefix does not exist, search shorter prefix */ max = mid - 1; mid = mid / 2; } @@ -6552,41 +6557,46 @@ uint32_t PDC_prefix_binary_search(char *prefix, char **target_prefix){ return -1; // No valid prefix found } -perr_t PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { +perr_t +PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) +{ FUNC_ENTER(NULL); - char *key_name; - char *target_prefix; - perr_t ret_value = SUCCEED; - hg_return_t hg_ret = 0; - hg_handle_t metadata_key_add_handle; - metadata_key_add_in_t in; - struct _pdc_client_lookup_args lookup_args; + char * key_name; + char * target_prefix; + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + hg_handle_t metadata_key_add_handle; + metadata_key_add_in_t in; + struct _pdc_client_lookup_args lookup_args; printf("PDC_metadata_key_add: Adding key %s to metadata\n", kvtag->name); - key_name = kvtag->name; - char * key_bin = string_to_binary(key_name); - char *prefix = malloc(strlen(key_bin) + 2); + key_name = kvtag->name; + char *key_bin = string_to_binary(key_name); + char *prefix = malloc(strlen(key_bin) + 2); strncpy(prefix, ROOT_BUCKET, 1); - strncpy(prefix+1, key_bin, strlen(key_bin)); - prefix[strlen(key_bin)+1] = '\0'; + strncpy(prefix + 1, key_bin, strlen(key_bin)); + prefix[strlen(key_bin) + 1] = '\0'; free(key_bin); uint32_t server_id = PDC_prefix_binary_search(prefix, &target_prefix); if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); - - HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_key_add_register_id_g, &metadata_key_add_handle); - - in.key = strdup(kvtag->name); - in.value = kvtag->value; + + HG_Create(send_context_g, pdc_server_info_g[server_id].addr, metadata_key_add_register_id_g, + &metadata_key_add_handle); + + in.key = strdup(kvtag->name); + in.value = kvtag->value; in.prefix = strdup(target_prefix); - in.size = kvtag->size; - - printf("PDC_metadata_key_add: Inserting key %s with prefix %s and value %s to server %d\n", in.key, in.prefix, in.value, server_id); - + in.size = kvtag->size; + + printf("PDC_metadata_key_add: Inserting key %s with prefix %s and value %s to server %d\n", in.key, + in.prefix, in.value, server_id); + hg_ret = HG_Forward(metadata_key_add_handle, metadata_key_add_rpc_cb, &lookup_args, &in); printf("HG_Forward returned %d\n", hg_ret); - if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not start HG_Forward"); + if (hg_ret != HG_SUCCESS) + PGOTO_ERROR(FAIL, "Could not start HG_Forward"); // Wait for response from server hg_atomic_set32(&atomic_work_todo_g, 1); @@ -6600,22 +6610,25 @@ perr_t PDC_metadata_key_add(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { FUNC_LEAVE(ret_value); } -perr_t PDC_metadata_key_delete(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) { +perr_t +PDC_metadata_key_delete(pdcid_t obj_id, pdc_kvtag_t *kvtag, int is_cont) +{ FUNC_ENTER(NULL); - char *key_name; - char *target_prefix; + char * key_name; + char * target_prefix; perr_t ret_value = SUCCEED; printf("PDC_metadata_key_delete: Adding key %s to metadata\n", kvtag->name); - key_name = kvtag->name; - char * key_bin = string_to_binary(key_name); - char *prefix = malloc(strlen(key_bin) + 2); + key_name = kvtag->name; + char *key_bin = string_to_binary(key_name); + char *prefix = malloc(strlen(key_bin) + 2); strncpy(prefix, ROOT_BUCKET, 1); - strncpy(prefix+1, key_bin, strlen(key_bin)); - prefix[strlen(key_bin)+1] = '\0'; + strncpy(prefix + 1, key_bin, strlen(key_bin)); + prefix[strlen(key_bin) + 1] = '\0'; free(key_bin); uint32_t server_id = PDC_prefix_binary_search(prefix, &target_prefix); - printf("PDC_metadata_key_delete: Inserting key %s with prefix %s to server %d\n", key_name, target_prefix, server_id); + printf("PDC_metadata_key_delete: Inserting key %s with prefix %s to server %d\n", key_name, target_prefix, + server_id); free(target_prefix); free(prefix); FUNC_LEAVE(ret_value); @@ -8673,7 +8686,8 @@ PDC_Client_search_obj_ref_through_dart_mpi(dart_hash_algo_t hash_algo, char *que } perr_t -PDC_Client_create_bucket(char *prefix){ +PDC_Client_create_bucket(char *prefix) +{ FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; @@ -8683,7 +8697,7 @@ PDC_Client_create_bucket(char *prefix){ struct _pdc_client_lookup_args lookup_args; uint64_t hash_value = prefix_hash(prefix); - uint32_t server_id = PDC_get_server_using_pht(hash_value); + uint32_t server_id = PDC_get_server_using_pht(hash_value); if (PDC_Client_try_lookup_server(server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); @@ -8694,10 +8708,11 @@ PDC_Client_create_bucket(char *prefix){ // Fill input structure in.prefix = prefix; - hg_ret = HG_Forward(metadata_create_bucket_handle, metadata_create_bucket_client_rpc_cb, &lookup_args, &in); + hg_ret = + HG_Forward(metadata_create_bucket_handle, metadata_create_bucket_client_rpc_cb, &lookup_args, &in); if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not start HG_Forward"); - // Wait for response from server + // Wait for response from server hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); // BLOCKING CALL @@ -8714,8 +8729,8 @@ metadata_create_bucket_client_rpc_cb(const struct hg_cb_info *callback_info) { FUNC_ENTER(NULL); - hg_return_t ret_value; - hg_handle_t handle = callback_info->info.forward.handle; + hg_return_t ret_value; + hg_handle_t handle = callback_info->info.forward.handle; struct _pdc_client_lookup_args *client_lookup_args = (struct _pdc_client_lookup_args *)callback_info->arg; /* Get output from server*/ @@ -8735,7 +8750,6 @@ metadata_create_bucket_client_rpc_cb(const struct hg_cb_info *callback_info) FUNC_LEAVE(ret_value); } - #endif /******************** Collective Object Selection Query Ends *******************************/ diff --git a/src/commons/collections/include/pdc_dllist.h b/src/commons/collections/include/pdc_dllist.h index 34022beef..50cc5c8b0 100644 --- a/src/commons/collections/include/pdc_dllist.h +++ b/src/commons/collections/include/pdc_dllist.h @@ -56,23 +56,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **/ // Function pointer typedefs -typedef void* DoublyLinkedListKey; -typedef void* DoublyLinkedListValue; +typedef void *DoublyLinkedListKey; +typedef void *DoublyLinkedListValue; typedef unsigned int (*DoublyLinkedListHashFunc)(DoublyLinkedListKey data); typedef int (*DoublyLinkedListEqualFunc)(DoublyLinkedListKey value1, DoublyLinkedListKey value2); typedef void (*DoublyLinkedListKeyFreeFunc)(DoublyLinkedListKey value); typedef struct DoublyLinkedListItem DoublyLinkedListItem; -typedef struct DoublyLinkedList DoublyLinkedList; - +typedef struct DoublyLinkedList DoublyLinkedList; DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func); -int dllist_insert(DoublyLinkedList *list, void *key, void *value); -int *dllist_search_key(DoublyLinkedList *list, void *key); -int *dllist_search_range(DoublyLinkedList *list, void *start_key, bool include_start, void *end_key, bool include_end); -int dllist_destroy(void *list); - /** +int dllist_insert(DoublyLinkedList *list, void *key, void *value); +int * dllist_search_key(DoublyLinkedList *list, void *key); +int *dllist_search_range(DoublyLinkedList *list, void *start_key, bool include_start, void *end_key, + bool include_end); +int dllist_destroy(void *list); +/** * -------------------------------------------------- * * For doubly-linked lists, the append and delete macros are O(1) diff --git a/src/commons/collections/include/pdc_pht.h b/src/commons/collections/include/pdc_pht.h index 6ffa66412..6c39e03f6 100644 --- a/src/commons/collections/include/pdc_pht.h +++ b/src/commons/collections/include/pdc_pht.h @@ -12,19 +12,19 @@ #define ROOT_PREFIX "#" typedef struct DoublyLinkedListItem { - char *key; - void *value; + char * key; + void * value; struct DoublyLinkedList *prev; /* needed for a doubly-linked list only */ struct DoublyLinkedList *next; /* needed for singly- or doubly-linked lists */ } DoublyLinkedListItem; typedef struct DoublyLinkedList { DoublyLinkedListItem *head; /* first element in list */ - unsigned int count; + unsigned int count; } DoublyLinkedList; -typedef void* PrefixTableKey; -typedef void* PrefixTableValue; +typedef void *PrefixTableKey; +typedef void *PrefixTableValue; /** * Hash function used to generate hash values for keys used in a hash @@ -61,20 +61,20 @@ struct _PrefixTableBucket { char *left; char *right; - char *prefix; + char * prefix; DoublyLinkedList *store; }; struct _PrefixTable { - HashTable *map; - size_t bucket_size; - size_t key_count; - PrefixTableHashFunc hash_cb; + HashTable * map; + size_t bucket_size; + size_t key_count; + PrefixTableHashFunc hash_cb; PrefixTableEqualFunc equal_cb; }; typedef struct _PrefixTableBucket PrefixTableBucket; -typedef struct _PrefixTable PrefixTable; +typedef struct _PrefixTable PrefixTable; /** * Initializes a new prefix table. @@ -84,7 +84,8 @@ typedef struct _PrefixTable PrefixTable; * @param equal_cb Function to compare keys for equality. * @return A pointer to the initialized prefix table, or NULL on failure. */ -PrefixTable* prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb); +PrefixTable *prefix_table_init(unsigned int bucket_size, PrefixTableHashFunc hash_cb, + PrefixTableEqualFunc equal_cb); /** * Inserts a key-value pair into the prefix table. @@ -117,7 +118,8 @@ int prefix_table_exact_search(PrefixTable *pht, void *key, void **result); * @param result Pointer to store the found values, or NULL if not found. * @return 0 on success, or a negative error code on failure. */ -int prefix_table_range_search(PrefixTable *pht, void *start_key, bool include_start, void *end_key, bool include_end, void **result); +int prefix_table_range_search(PrefixTable *pht, void *start_key, bool include_start, void *end_key, + bool include_end, void **result); /** * Counts the number of keys in the prefix table. @@ -141,7 +143,7 @@ void prefix_table_destroy(PrefixTable *pht); * @param equal_cb Equality comparison callback. * @return A pointer to the initialized bucket, or NULL on failure. */ -PrefixTableBucket* prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb); +PrefixTableBucket *prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb); /** * Checks if a bucket is a leaf node. diff --git a/src/commons/collections/include/pdc_set.h b/src/commons/collections/include/pdc_set.h index a6818b82e..0c753ca4a 100644 --- a/src/commons/collections/include/pdc_set.h +++ b/src/commons/collections/include/pdc_set.h @@ -189,8 +189,7 @@ int set_query(Set *set, SetValue data); * @return Zero on success, or -1 if it was not possible to * allocate memory for the new entries. */ -int set_copy(Set *target, Set* source); - +int set_copy(Set *target, Set *source); /** * Retrieve the number of entries in a set diff --git a/src/commons/collections/pdc_dllist.c b/src/commons/collections/pdc_dllist.c index de51f4fa2..b00caf765 100644 --- a/src/commons/collections/pdc_dllist.c +++ b/src/commons/collections/pdc_dllist.c @@ -3,67 +3,82 @@ // Now define the actual structures struct DoublyLinkedListItem { - void * key; - void * value; - struct DoublyLinkedListItem *prev; // Use 'struct' keyword - struct DoublyLinkedListItem *next; // Use 'struct' keyword + void * key; + void * value; + struct DoublyLinkedListItem *prev; // Use 'struct' keyword + struct DoublyLinkedListItem *next; // Use 'struct' keyword }; struct DoublyLinkedList { - DoublyLinkedListItem * head; - DoublyLinkedListHashFunc hash_func; + DoublyLinkedListItem * head; + DoublyLinkedListHashFunc hash_func; DoublyLinkedListEqualFunc equal_func; - unsigned int count; + unsigned int count; }; -DoublyLinkedList *dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func){ +DoublyLinkedList * +dllist_init(DoublyLinkedListHashFunc hash_func, DoublyLinkedListEqualFunc equal_func) +{ DoublyLinkedList *list = malloc(sizeof(DoublyLinkedList)); if (!list) { return NULL; // Memory allocation failed } - list->hash_func = hash_func; + list->hash_func = hash_func; list->equal_func = equal_func; - list->head = NULL; - list->count = 0; + list->head = NULL; + list->count = 0; return list; } -int dllist_insert(DoublyLinkedList *list, void *key, void *value){ +int +dllist_insert(DoublyLinkedList *list, void *key, void *value) +{ DoublyLinkedListItem *item = malloc(sizeof(DoublyLinkedListItem)); - item->key = key; - item->value = value; + item->key = key; + item->value = value; DL_APPEND(list->head, item); list->count++; return 0; } -int *dllist_search_key(DoublyLinkedList *list, void *key){ - DoublyLinkedListItem *elt = NULL; +int * +dllist_search_key(DoublyLinkedList *list, void *key) +{ + DoublyLinkedListItem * elt = NULL; DoublyLinkedListEqualFunc key_cmp_func = list->equal_func; - DL_FOREACH(list->head, elt) { - if(key_cmp_func(elt->key, key) == 0) break; + DL_FOREACH(list->head, elt) + { + if (key_cmp_func(elt->key, key) == 0) + break; } return elt != NULL && key_cmp_func(elt->key, key) == 0 ? elt->value : NULL; } -int *dllist_search_range(DoublyLinkedList * list, void *start_key, bool include_start, void *end_key, bool include_end){ - DoublyLinkedListItem *elt = NULL; - Set *value_set = set_new(NULL, NULL); +int * +dllist_search_range(DoublyLinkedList *list, void *start_key, bool include_start, void *end_key, + bool include_end) +{ + DoublyLinkedListItem * elt = NULL; + Set * value_set = set_new(NULL, NULL); DoublyLinkedListEqualFunc key_cmp_func = list->equal_func; - DL_FOREACH(list->head, elt) { + DL_FOREACH(list->head, elt) + { int start_dist = key_cmp_func(elt->key, start_key); - int end_dist = key_cmp_func(elt->key, end_key); - if(start_dist >= 0 && end_dist <= 0){ + int end_dist = key_cmp_func(elt->key, end_key); + if (start_dist >= 0 && end_dist <= 0) { set_copy(value_set, elt->value); } } return value_set; } -int dllist_destroy(void *ptr){ - DoublyLinkedList *list = (DoublyLinkedList*)ptr; - DoublyLinkedListItem *elt = NULL; - DoublyLinkedListItem *tmp = NULL; - DL_FOREACH_SAFE(list->head, elt, tmp) { +int +dllist_destroy(void *ptr) +{ + DoublyLinkedList * list = (DoublyLinkedList *)ptr; + DoublyLinkedListItem *elt = NULL; + DoublyLinkedListItem *tmp = NULL; + DL_FOREACH_SAFE(list->head, elt, tmp) + { DL_DELETE(list->head, elt); free(elt->value); free(elt->key); diff --git a/src/commons/collections/pdc_pht.c b/src/commons/collections/pdc_pht.c index ae594c87b..96c6c7e1d 100644 --- a/src/commons/collections/pdc_pht.c +++ b/src/commons/collections/pdc_pht.c @@ -1,43 +1,46 @@ #include "pdc_pht.h" - #ifdef __cplusplus extern "C" { #endif - - -PrefixTable* prefix_table_init(unsigned int bucket_size, HashTableHashFunc hash_cb, HashTableEqualFunc equal_cb){ +PrefixTable * +prefix_table_init(unsigned int bucket_size, HashTableHashFunc hash_cb, HashTableEqualFunc equal_cb) +{ PrefixTable *pht = (PrefixTable *)malloc(sizeof(PrefixTable)); if (!pht) { return NULL; // Memory allocation failed } pht->bucket_size = bucket_size; - pht->key_count = 0; - pht->map = hash_table_new(hash_cb, equal_cb); + pht->key_count = 0; + pht->map = hash_table_new(hash_cb, equal_cb); if (!pht->map) { free(pht); return NULL; // Hash table creation failed } - pht->hash_cb = hash_cb; + pht->hash_cb = hash_cb; pht->equal_cb = equal_cb; return pht; } -PrefixTableBucket* prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb){ +PrefixTableBucket * +prefix_table_bucket_init(PrefixTableHashFunc hash_cb, PrefixTableEqualFunc equal_cb) +{ PrefixTableBucket *bucket = (PrefixTableBucket *)malloc(sizeof(PrefixTableBucket)); if (!bucket) { return NULL; // Memory allocation failed } bucket->isLeaf = true; bucket->parent = NULL; - bucket->left = NULL; - bucket->right = NULL; + bucket->left = NULL; + bucket->right = NULL; bucket->prefix = NULL; - bucket->store = dllist_init(hash_cb, equal_cb); + bucket->store = dllist_init(hash_cb, equal_cb); return bucket; } -int prefix_table_insert(PrefixTable *pht, void *key, void *value){ +int +prefix_table_insert(PrefixTable *pht, void *key, void *value) +{ if (!pht || !key || !value) { return -1; // Invalid parameters } @@ -46,15 +49,21 @@ int prefix_table_insert(PrefixTable *pht, void *key, void *value){ return 0; } -void prefix_table_destroy(PrefixTable *pht){ +void +prefix_table_destroy(PrefixTable *pht) +{ hash_table_free(pht->map); free(pht); } -void pht_add_to_store(PrefixTable *pht, void *key, void *value){ +void +pht_add_to_store(PrefixTable *pht, void *key, void *value) +{ hash_table_insert(pht->map, key, value); } -bool prefix_table_bucket_is_leaf(PrefixTableBucket *bucket){ +bool +prefix_table_bucket_is_leaf(PrefixTableBucket *bucket) +{ return bucket->isLeaf; } diff --git a/src/commons/collections/pdc_set.c b/src/commons/collections/pdc_set.c index 1f46de85a..6086ee03f 100644 --- a/src/commons/collections/pdc_set.c +++ b/src/commons/collections/pdc_set.c @@ -405,7 +405,9 @@ set_num_entries(Set *set) FUNC_LEAVE(set->entries); } -int set_copy(Set *target, Set* source){ +int +set_copy(Set *target, Set *source) +{ SetIterator iterator; SetValue value; set_iterate(source, &iterator); diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index f49eac643..02cedc869 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -521,18 +521,18 @@ typedef struct { } metadata_delete_by_id_out_t; typedef struct { - char* prefix; + char *prefix; } metadata_check_prefix_in_t; typedef struct { - int32_t ret; - bool found; - bool leaf; + int32_t ret; + bool found; + bool leaf; uint32_t server_id; } metadata_check_prefix_out_t; typedef struct { - char* prefix; + char *prefix; } metadata_create_bucket_in_t; typedef struct { @@ -540,9 +540,9 @@ typedef struct { } metadata_create_bucket_out_t; typedef struct { - char* prefix; - char* key; - void* value; + char * prefix; + char * key; + void * value; uint32_t size; } metadata_key_add_in_t; @@ -1762,11 +1762,12 @@ hg_proc_metadata_add_tag_in_t(hg_proc_t proc, void *data) } static HG_INLINE hg_return_t -hg_proc_metadata_check_prefix_in_t(hg_proc_t proc, void *data) { +hg_proc_metadata_check_prefix_in_t(hg_proc_t proc, void *data) +{ FUNC_ENTER(NULL); - hg_return_t ret; - metadata_check_prefix_in_t *struct_data = (metadata_check_prefix_in_t *)data; + hg_return_t ret; + metadata_check_prefix_in_t *struct_data = (metadata_check_prefix_in_t *)data; ret = hg_proc_hg_string_t(proc, &struct_data->prefix); if (ret != HG_SUCCESS) { @@ -1777,11 +1778,12 @@ hg_proc_metadata_check_prefix_in_t(hg_proc_t proc, void *data) { } static HG_INLINE hg_return_t -hg_proc_metadata_check_prefix_out_t(hg_proc_t proc, void *data) { +hg_proc_metadata_check_prefix_out_t(hg_proc_t proc, void *data) +{ FUNC_ENTER(NULL); - hg_return_t ret; - metadata_check_prefix_out_t *struct_data = (metadata_check_prefix_out_t *)data; + hg_return_t ret; + metadata_check_prefix_out_t *struct_data = (metadata_check_prefix_out_t *)data; ret = hg_proc_hg_bool_t(proc, &struct_data->found); if (ret != HG_SUCCESS) { @@ -1804,11 +1806,12 @@ hg_proc_metadata_check_prefix_out_t(hg_proc_t proc, void *data) { } static HG_INLINE hg_return_t -hg_proc_metadata_create_bucket_in_t(hg_proc_t proc, void *data) { +hg_proc_metadata_create_bucket_in_t(hg_proc_t proc, void *data) +{ FUNC_ENTER(NULL); - hg_return_t ret; - metadata_create_bucket_in_t *struct_data = (metadata_create_bucket_in_t *)data; + hg_return_t ret; + metadata_create_bucket_in_t *struct_data = (metadata_create_bucket_in_t *)data; printf("In hg_proc_metadata_create_bucket_in_t %s\n", struct_data->prefix); ret = hg_proc_hg_string_t(proc, &struct_data->prefix); @@ -1820,11 +1823,12 @@ hg_proc_metadata_create_bucket_in_t(hg_proc_t proc, void *data) { } static HG_INLINE hg_return_t -hg_proc_metadata_create_bucket_out_t(hg_proc_t proc, void *data) { +hg_proc_metadata_create_bucket_out_t(hg_proc_t proc, void *data) +{ FUNC_ENTER(NULL); - hg_return_t ret; - metadata_create_bucket_out_t *struct_data = (metadata_create_bucket_out_t *)data; + hg_return_t ret; + metadata_create_bucket_out_t *struct_data = (metadata_create_bucket_out_t *)data; ret = hg_proc_int32_t(proc, &struct_data->ret); if (ret != HG_SUCCESS) { @@ -1901,23 +1905,27 @@ static HG_INLINE hg_return_t hg_proc_metadata_key_add_in_t(hg_proc_t proc, void *data) { FUNC_ENTER(NULL); - hg_return_t ret; - metadata_key_add_in_t *struct_data = (metadata_key_add_in_t *)data; + hg_return_t ret; + metadata_key_add_in_t *struct_data = (metadata_key_add_in_t *)data; ret = hg_proc_hg_string_t(proc, &struct_data->key); - if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + if (ret != HG_SUCCESS) + FUNC_LEAVE(ret); printf("In hg_proc_metadata_key_add_in_t Key:%s\n", struct_data->key); ret = hg_proc_hg_string_t(proc, &struct_data->prefix); - if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + if (ret != HG_SUCCESS) + FUNC_LEAVE(ret); printf("In hg_proc_metadata_key_add_in_t Prefix:%s\n", struct_data->prefix); - + ret = hg_proc_raw(proc, struct_data->value, struct_data->size); - if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + if (ret != HG_SUCCESS) + FUNC_LEAVE(ret); printf("In hg_proc_metadata_key_add_in_t Value:%p\n", struct_data->value); ret = hg_proc_uint32_t(proc, &struct_data->size); - if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + if (ret != HG_SUCCESS) + FUNC_LEAVE(ret); printf("In hg_proc_metadata_key_add_in_t Size:%d\n", struct_data->size); FUNC_LEAVE(ret); @@ -1928,11 +1936,12 @@ hg_proc_metadata_key_add_out_t(hg_proc_t proc, void *data) { FUNC_ENTER(NULL); - hg_return_t ret; + hg_return_t ret; metadata_key_add_out_t *struct_data = (metadata_key_add_out_t *)data; ret = hg_proc_int32_t(proc, &struct_data->ret); - if (ret != HG_SUCCESS) FUNC_LEAVE(ret); + if (ret != HG_SUCCESS) + FUNC_LEAVE(ret); FUNC_LEAVE(ret); } @@ -4908,7 +4917,7 @@ perr_t PDC_create_shm_segment_ind(uint64_t size, char *shm_addr, void **buf); * * \return Binary representation of the string */ -char* string_to_binary(void *ptr); +char *string_to_binary(void *ptr); /** * Calculate the hash value of a string @@ -4986,5 +4995,5 @@ perr_t PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint6 int is_write); uint32_t PDC_get_server_using_pht(uint64_t key_hash); -perr_t PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out); +perr_t PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_out_t *out); #endif /* PDC_CLIENT_SERVER_COMMON_H */ diff --git a/src/server/include/pdc_server.h b/src/server/include/pdc_server.h index 7d80460a0..9039c98c3 100644 --- a/src/server/include/pdc_server.h +++ b/src/server/include/pdc_server.h @@ -210,7 +210,7 @@ hg_return_t PDC_Server_recv_shm_cb(const struct hg_cb_info *callback_info); * \param argv[IN] Command line arguments * \return Non-negative on success/Negative on failure */ -int server_run(int argc, char *argv[]); -perr_t PDC_Server2Server_create_bucket(char *prefix, uint32_t *server_id); +int server_run(int argc, char *argv[]); +perr_t PDC_Server2Server_create_bucket(char *prefix, uint32_t *server_id); hg_return_t metadata_create_bucket_server_rpc_cb(const struct hg_cb_info *callback_info); #endif /* PDC_SERVER_H */ diff --git a/src/server/include/pdc_server_metadata.h b/src/server/include/pdc_server_metadata.h index d853856fa..420787dd5 100644 --- a/src/server/include/pdc_server_metadata.h +++ b/src/server/include/pdc_server_metadata.h @@ -381,5 +381,4 @@ perr_t PDC_free_cont_hash_table(); */ perr_t PDC_Server_add_kvtag(metadata_add_kvtag_in_t *in, metadata_add_tag_out_t *out); - #endif /* PDC_SERVER_METADATA_H */ diff --git a/src/server/include/pdc_server_metadata_pht.h b/src/server/include/pdc_server_metadata_pht.h index 315dbfac5..e0093a46c 100644 --- a/src/server/include/pdc_server_metadata_pht.h +++ b/src/server/include/pdc_server_metadata_pht.h @@ -13,7 +13,7 @@ #define PHT_BUCKET_SIZE 1024 -void PDC_Server_metadata_pht_init(uint32_t num_server, uint32_t server_id); +void PDC_Server_metadata_pht_init(uint32_t num_server, uint32_t server_id); perr_t PDC_Server_add_metadata(); /** * Create a bucket with the given prefix diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index 8d058c343..a6099dcbf 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -55,10 +55,10 @@ #include "pdc_timing.h" #include "pdc_server_region_cache.h" -int pdc_server_size_g = 1; -static hg_context_t * send_context_g = NULL; -hg_id_t metadata_create_bucket_register_id_g; -static hg_atomic_int32_t atomic_work_todo_g; +int pdc_server_size_g = 1; +static hg_context_t * send_context_g = NULL; +hg_id_t metadata_create_bucket_register_id_g; +static hg_atomic_int32_t atomic_work_todo_g; #ifdef ENABLE_MULTITHREAD hg_thread_mutex_t insert_metadata_mutex_g = HG_THREAD_MUTEX_INITIALIZER; @@ -229,20 +229,24 @@ PDC_get_server_by_obj_id(uint64_t obj_id, int n_server) FUNC_LEAVE(ret_value); } -uint64_t prefix_hash(const char *key){ +uint64_t +prefix_hash(const char *key) +{ uint64_t hash = 0; - uint64_t len = strlen(key); - int i = 1; + uint64_t len = strlen(key); + int i = 1; while (i < len) { - if (*key == '\0') break; + if (*key == '\0') + break; if (*key == '#') { *key++; continue; }; if (*key == '0') { - hash = (hash*193) + (i * 90); - } else { - hash = (hash*193) + (i * 270); + hash = (hash * 193) + (i * 90); + } + else { + hash = (hash * 193) + (i * 270); } *key++; i++; @@ -250,17 +254,19 @@ uint64_t prefix_hash(const char *key){ return hash; } -char* string_to_binary(void *ptr){ - char *str = (char *)ptr; - size_t len = strlen(str); - char *binStr = malloc(len*8 + 1); - for (int i = 0; i < len; i++){ +char * +string_to_binary(void *ptr) +{ + char * str = (char *)ptr; + size_t len = strlen(str); + char * binStr = malloc(len * 8 + 1); + for (int i = 0; i < len; i++) { char c = str[i]; - for (int j=7; j>=0; j--) { - binStr[(i*8) + 7 - j] = (c & (1 << j)) ? '1' : '0'; + for (int j = 7; j >= 0; j--) { + binStr[(i * 8) + 7 - j] = (c & (1 << j)) ? '1' : '0'; } } - binStr[len*8] = '\0'; + binStr[len * 8] = '\0'; return binStr; } @@ -1013,21 +1019,22 @@ PDC_Server_get_kvtag(metadata_get_kvtag_in_t *in ATTRIBUTE(unused), } perr_t -PDC_Server_check_prefix(metadata_check_prefix_in_t *in ATTRIBUTE(unused), - metadata_check_prefix_out_t *out ATTRIBUTE(unused)) +PDC_Server_check_prefix(metadata_check_prefix_in_t *in ATTRIBUTE(unused), + metadata_check_prefix_out_t *out ATTRIBUTE(unused)) { return SUCCEED; } perr_t -PDC_Server_create_bucket(metadata_create_bucket_in_t *in ATTRIBUTE(unused), - metadata_create_bucket_out_t *out ATTRIBUTE(unused)){ +PDC_Server_create_bucket(metadata_create_bucket_in_t *in ATTRIBUTE(unused), + metadata_create_bucket_out_t *out ATTRIBUTE(unused)) +{ return SUCCEED; } perr_t -PDC_Server_metadata_key_add(metadata_key_add_in_t *in ATTRIBUTE(unused), - metadata_key_add_out_t *out ATTRIBUTE(unused)) +PDC_Server_metadata_key_add(metadata_key_add_in_t *in ATTRIBUTE(unused), + metadata_key_add_out_t *out ATTRIBUTE(unused)) { return SUCCEED; } @@ -1912,11 +1919,11 @@ HG_TEST_RPC_CB(metadata_add_kvtag, handle) HG_TEST_RPC_CB(metadata_check_prefix, handle) { - FUNC_ENTER(NULL); + FUNC_ENTER(NULL); hg_return_t ret_value = HG_SUCCESS; metadata_check_prefix_in_t in; - metadata_check_prefix_out_t out; + metadata_check_prefix_out_t out; HG_Get_input(handle, &in); if (strcmp(in.prefix, "PDC_NOOP") != 0) { @@ -1937,18 +1944,19 @@ HG_TEST_RPC_CB(metadata_check_prefix, handle) HG_TEST_RPC_CB(metadata_key_add, handle) { - FUNC_ENTER(NULL); + FUNC_ENTER(NULL); - hg_return_t ret_value = HG_SUCCESS; - metadata_key_add_in_t in; - metadata_key_add_out_t out; + hg_return_t ret_value = HG_SUCCESS; + metadata_key_add_in_t in; + metadata_key_add_out_t out; printf("metadata_key_add %s\n", &in.prefix); HG_Get_input(handle, &in); if (strcmp(in.prefix, "PDC_NOOP") != 0) { printf("PDC_Server_metadata_key_add\n"); PDC_Server_metadata_key_add(&in, &out); - } else { + } + else { LOG_INFO("Received NOOP\n"); out.ret = 1; } @@ -1963,11 +1971,11 @@ HG_TEST_RPC_CB(metadata_key_add, handle) HG_TEST_RPC_CB(metadata_create_bucket, handle) { - FUNC_ENTER(NULL); + FUNC_ENTER(NULL); - hg_return_t ret_value = HG_SUCCESS; - metadata_create_bucket_in_t in; - metadata_create_bucket_out_t out; + hg_return_t ret_value = HG_SUCCESS; + metadata_create_bucket_in_t in; + metadata_create_bucket_out_t out; HG_Get_input(handle, &in); if (strcmp(in.prefix, "PDC_NOOP") != 0) { @@ -6302,9 +6310,11 @@ PDC_FUNC_DECLARE_REGISTER(metadata_add_tag) PDC_FUNC_DECLARE_REGISTER(send_rpc) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_del_kvtag, metadata_get_kvtag_in_t, metadata_add_tag_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_add_kvtag, metadata_add_kvtag_in_t, metadata_add_tag_out_t) -PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_check_prefix, metadata_check_prefix_in_t, metadata_check_prefix_out_t) +PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_check_prefix, metadata_check_prefix_in_t, + metadata_check_prefix_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_key_add, metadata_key_add_in_t, metadata_key_add_out_t) -PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_create_bucket, metadata_create_bucket_in_t, metadata_create_bucket_out_t) +PDC_FUNC_DECLARE_REGISTER_IN_OUT(metadata_create_bucket, metadata_create_bucket_in_t, + metadata_create_bucket_out_t) PDC_FUNC_DECLARE_REGISTER(metadata_get_kvtag) PDC_FUNC_DECLARE_REGISTER(metadata_update) PDC_FUNC_DECLARE_REGISTER(metadata_delete_by_id) @@ -7069,8 +7079,9 @@ PDCselection_print_all(pdc_selection_t *sel) } uint32_t -PDC_get_server_using_pht(uint64_t key_hash) { - int ring_size = pdc_server_size_g * RING_ARC_SIZE; +PDC_get_server_using_pht(uint64_t key_hash) +{ + int ring_size = pdc_server_size_g * RING_ARC_SIZE; uint64_t mapped_hash = key_hash % ring_size; for (int i = 0; i < pdc_server_size_g; i++) { @@ -7082,5 +7093,5 @@ PDC_get_server_using_pht(uint64_t key_hash) { } // Wrap around - return 0; + return 0; } \ No newline at end of file diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index a31c6f631..627b4936f 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -2126,39 +2126,42 @@ PDC_Server_get_env() } perr_t -PDC_Server2Server_create_bucket(char *prefix, uint32_t *server_id) { +PDC_Server2Server_create_bucket(char *prefix, uint32_t *server_id) +{ FUNC_ENTER(NULL); - perr_t ret_value = SUCCEED; - hg_return_t hg_ret = 0; - hg_handle_t metadata_create_bucket_handle; - metadata_create_bucket_in_t in; - metadata_create_bucket_out_t out; + perr_t ret_value = SUCCEED; + hg_return_t hg_ret = 0; + hg_handle_t metadata_create_bucket_handle; + metadata_create_bucket_in_t in; + metadata_create_bucket_out_t out; uint64_t hash_value = prefix_hash(prefix); - *server_id = PDC_get_server_using_pht(hash_value); + *server_id = PDC_get_server_using_pht(hash_value); in.prefix = strdup(prefix); - if (*server_id == pdc_server_rank_g){ + if (*server_id == pdc_server_rank_g) { ret_value = PDC_Server_create_bucket(&in, &out); - if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Failed to get local storage location"); - } else { + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to get local storage location"); + } + else { if (PDC_Server_lookup_server_id(*server_id) != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); - - hg_ret = HG_Create(hg_context_g, pdc_remote_server_info_g[*(uint32_t *)server_id].addr, metadata_create_bucket_register_id_g, - &metadata_create_bucket_handle); - + + hg_ret = HG_Create(hg_context_g, pdc_remote_server_info_g[*(uint32_t *)server_id].addr, + metadata_create_bucket_register_id_g, &metadata_create_bucket_handle); + hg_ret = HG_Forward(metadata_create_bucket_handle, metadata_create_bucket_server_rpc_cb, NULL, &in); - - if (hg_ret != HG_SUCCESS){ + + if (hg_ret != HG_SUCCESS) { HG_Destroy(metadata_create_bucket_handle); PGOTO_ERROR(FAIL, "Could not start HG_Forward"); } - + if (hg_ret != SUCCEED) LOG_ERROR("Add create_bucket server2server NOT successful"); - + HG_Destroy(metadata_create_bucket_handle); } done: @@ -2170,14 +2173,14 @@ metadata_create_bucket_server_rpc_cb(const struct hg_cb_info *callback_info) { FUNC_ENTER(NULL); - hg_return_t ret_value; - hg_handle_t handle = callback_info->info.forward.handle; - metadata_create_bucket_out_t *result = (metadata_create_bucket_out_t *)callback_info->arg; + hg_return_t ret_value; + hg_handle_t handle = callback_info->info.forward.handle; + metadata_create_bucket_out_t *result = (metadata_create_bucket_out_t *)callback_info->arg; /* Get output from server*/ metadata_create_bucket_out_t output; ret_value = HG_Get_output(handle, &output); - result = &output; + result = &output; if (ret_value != HG_SUCCESS) { PGOTO_ERROR(HG_OTHER_ERROR, "Error with HG_Get_output"); diff --git a/src/server/pdc_server_metadata_pht.c b/src/server/pdc_server_metadata_pht.c index be5bbdbf1..ca6f73299 100644 --- a/src/server/pdc_server_metadata_pht.c +++ b/src/server/pdc_server_metadata_pht.c @@ -54,13 +54,14 @@ #include "pdc_logger.h" #include "pdc_pht.h" - uint32_t metadata_server_id_g = 0; uint32_t metadata_num_server_g = 0; PrefixTable *metadata_pht_key_g = NULL; -unsigned int pht_string_hash(void *vlocation) { +unsigned int +pht_string_hash(void *vlocation) +{ unsigned int result = 5381; unsigned char *p; @@ -74,10 +75,12 @@ unsigned int pht_string_hash(void *vlocation) { return result; } -int pht_string_comparator(const void *key1, const void *key2){ - char *ch1 = (char *) key1; - char *ch2 = (char *) key2; - int diff = strcmp(ch1, ch2); +int +pht_string_comparator(const void *key1, const void *key2) +{ + char *ch1 = (char *)key1; + char *ch2 = (char *)key2; + int diff = strcmp(ch1, ch2); return diff < 0 ? -1 : diff > 0 ? 1 : 0; } @@ -89,9 +92,9 @@ PDC_Server_metadata_pht_init(uint32_t num_server, uint32_t server_id) { FUNC_ENTER(NULL); - metadata_server_id_g = num_server; - metadata_num_server_g = server_id; - metadata_pht_key_g = prefix_table_init(PHT_BUCKET_SIZE, string_hash, string_equal); + metadata_server_id_g = num_server; + metadata_num_server_g = server_id; + metadata_pht_key_g = prefix_table_init(PHT_BUCKET_SIZE, string_hash, string_equal); FUNC_LEAVE_VOID(); } @@ -121,11 +124,11 @@ PDC_Server_check_prefix(metadata_check_prefix_in_t *in, metadata_check_prefix_ou printf("PDC_Server_check_prefix: metadata_pht_key_g = %d\n", metadata_pht_key_g->key_count); HashTableValue *value = hash_table_lookup(metadata_pht_key_g->map, in->prefix); printf("PDC_Server_check_prefix: value = %p\n", value); - PrefixTableBucket *bucket = (PrefixTableBucket *) value; + PrefixTableBucket *bucket = (PrefixTableBucket *)value; printf("PDC_Server_check_prefix: isLeaf = %d\n", bucket->isLeaf); - out->found = (bucket != NULL) ? 1 : 0; - out->leaf = prefix_table_bucket_is_leaf(bucket); - out->ret = 1; + out->found = (bucket != NULL) ? 1 : 0; + out->leaf = prefix_table_bucket_is_leaf(bucket); + out->ret = 1; out->server_id = metadata_server_id_g; done: @@ -195,7 +198,7 @@ PDC_Server_create_bucket(metadata_create_bucket_in_t *in, metadata_create_bucket goto done; // already exists } PrefixTableBucket *new_bucket = prefix_table_bucket_init(pht_string_hash, pht_string_comparator); - new_bucket->prefix = strdup(in->prefix); + new_bucket->prefix = strdup(in->prefix); hash_table_insert(metadata_pht_key_g->map, new_bucket->prefix, new_bucket); printf("PDC_Server_create_bucket: new_bucket = %p\n", new_bucket); out->ret = 1; @@ -233,7 +236,6 @@ PDC_Server_create_bucket(metadata_create_bucket_in_t *in, metadata_create_bucket FUNC_LEAVE(ret_value); } - perr_t PDC_Server_bucket_split(metadata_create_bucket_in_t *in, metadata_create_bucket_out_t *out) { @@ -267,7 +269,7 @@ PDC_Server_bucket_split(metadata_create_bucket_in_t *in, metadata_create_bucket_ goto done; // already exists } PrefixTableBucket *new_bucket = prefix_table_bucket_init(pht_string_hash, pht_string_comparator); - new_bucket->prefix = strdup(in->prefix); + new_bucket->prefix = strdup(in->prefix); hash_table_insert(metadata_pht_key_g->map, new_bucket->prefix, new_bucket); printf("PDC_Server_create_bucket: new_bucket = %p\n", new_bucket); out->ret = 1; @@ -330,7 +332,7 @@ PDC_Server_metadata_key_add(metadata_key_add_in_t *in, metadata_key_add_out_t *o printf("PDC_Server_metadata_key_add: metadata_pht_key_g = %d\n", metadata_pht_key_g->key_count); HashTableValue *value = hash_table_lookup(metadata_pht_key_g->map, in->prefix); printf("PDC_Server_metadata_key_add: value = %p\n", value); - PrefixTableBucket *bucket = (PrefixTableBucket *) value; + PrefixTableBucket *bucket = (PrefixTableBucket *)value; if (bucket == NULL) { perror("bucket is NULL"); ret_value = FAIL; @@ -340,8 +342,10 @@ PDC_Server_metadata_key_add(metadata_key_add_in_t *in, metadata_key_add_out_t *o Set *prevSet = dllist_search_key(bucket->store, in->key); if (prevSet != NULL) { set_insert(prevSet, in->value); - } else { - if (bucket->store->count >= PHT_BUCKET_SIZE) pht_bucket_split(bucket); + } + else { + if (bucket->store->count >= PHT_BUCKET_SIZE) + pht_bucket_split(bucket); Set *newSet = set_new(metadata_pht_key_g->hash_cb, metadata_pht_key_g->equal_cb); set_insert(newSet, in->value); dllist_insert(bucket->store, strdup(in->key), newSet); @@ -380,8 +384,10 @@ PDC_Server_metadata_key_add(metadata_key_add_in_t *in, metadata_key_add_out_t *o FUNC_LEAVE(ret_value); } -int pht_bucket_split(PrefixTableBucket *bucket){ - char *left_prefix = malloc(sizeof(char) * (strlen(bucket->prefix) + 2)); +int +pht_bucket_split(PrefixTableBucket *bucket) +{ + char *left_prefix = malloc(sizeof(char) * (strlen(bucket->prefix) + 2)); char *right_prefix = malloc(sizeof(char) * (strlen(bucket->prefix) + 2)); sprintf(left_prefix, "%s0", bucket->prefix); sprintf(right_prefix, "%s1", bucket->prefix); @@ -389,32 +395,36 @@ int pht_bucket_split(PrefixTableBucket *bucket){ PDC_Server2Server_create_bucket(left_prefix, &left_server_id); PDC_Server2Server_create_bucket(right_prefix, &right_server_id); bucket->isLeaf = false; - bucket->left = left_prefix; - bucket->right = right_prefix; + bucket->left = left_prefix; + bucket->right = right_prefix; DoublyLinkedListItem *elt = NULL; - DL_FOREACH(bucket->store->head, elt) { - char key_prefix = string_to_binary(elt->key); - bool is_left = strcmp(key_prefix, left_prefix) == 0; - bool is_right = strcmp(key_prefix, right_prefix) == 0; - uint32_t server_id = is_left ? left_server_id : right_server_id; - if (is_left || is_right){ + DL_FOREACH(bucket->store->head, elt) + { + char key_prefix = string_to_binary(elt->key); + bool is_left = strcmp(key_prefix, left_prefix) == 0; + bool is_right = strcmp(key_prefix, right_prefix) == 0; + uint32_t server_id = is_left ? left_server_id : right_server_id; + if (is_left || is_right) { if (server_id == metadata_server_id_g) { - PrefixTableBucket *child_bucket = hash_table_lookup(metadata_pht_key_g->map, is_left ? left_prefix : right_prefix); - if (child_bucket == NULL) return -1; // error + PrefixTableBucket *child_bucket = + hash_table_lookup(metadata_pht_key_g->map, is_left ? left_prefix : right_prefix); + if (child_bucket == NULL) + return -1; // error dllist_insert(child_bucket->store, elt->key, elt->value); - } else { - metadata_key_add_in_t add_in; + } + else { + metadata_key_add_in_t add_in; metadata_key_add_out_t add_out; add_in.prefix = is_left ? left_prefix : right_prefix; - add_in.key = elt->key; - add_in.value = elt->value; - add_in.size = sizeof(Set *); - //PDC_Client_metadata_key_add(server_id, &add_in, &add_out); + add_in.key = elt->key; + add_in.value = elt->value; + add_in.size = sizeof(Set *); + // PDC_Client_metadata_key_add(server_id, &add_in, &add_out); } } } - //dllist_destroy(bucket->store); + // dllist_destroy(bucket->store); bucket->store = NULL; return 0; } \ No newline at end of file diff --git a/src/tests/pht/pht_add_key.c b/src/tests/pht/pht_add_key.c index 9b6a18bf5..4f08fefc8 100644 --- a/src/tests/pht/pht_add_key.c +++ b/src/tests/pht/pht_add_key.c @@ -46,7 +46,7 @@ main() // create a pdc TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); - // create a container property + // create a container property TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", "Call to PDCprop_create failed"); // create a container @@ -80,7 +80,8 @@ main() kvtag3.size = sizeof(double); TASSERT(PDC_metadata_key_add(obj1, &kvtag1, false) >= 0, - "Call to PDC_metadata_key_add succeeded for obj1", "Call to PDC_metadata_key_add failed for obj1"); + "Call to PDC_metadata_key_add succeeded for obj1", + "Call to PDC_metadata_key_add failed for obj1"); // close pdc TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); diff --git a/src/tests/pht/pht_create_bucket.c b/src/tests/pht/pht_create_bucket.c index 674a478f8..851ca6bda 100644 --- a/src/tests/pht/pht_create_bucket.c +++ b/src/tests/pht/pht_create_bucket.c @@ -33,13 +33,13 @@ int main() { - pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; - pdc_kvtag_t kvtag1, kvtag2, kvtag3; - char * v1 = "value1"; - int v2 = 2; - double v3 = 3.45; - psize_t value_size; - int ret_value = SUCCEED; + pdcid_t pdc, cont_prop, cont, obj_prop1, obj_prop2, obj1, obj2; + pdc_kvtag_t kvtag1, kvtag2, kvtag3; + char * v1 = "value1"; + int v2 = 2; + double v3 = 3.45; + psize_t value_size; + int ret_value = SUCCEED; // create a pdc TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed");