From a923304677983b5ccf17031c7ee7ab4c8874db68 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 2 May 2024 09:57:31 -0400 Subject: [PATCH 1/2] WIP --- Makefile | 8 ++++---- include/sparsemap.h | 6 ++++++ src/sparsemap.c | 40 ++++++++++++++++++++++++++++++++++++++++ tests/test.c | 4 ++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 94aa600..440aff4 100644 --- a/Makefile +++ b/Makefile @@ -5,16 +5,16 @@ SHARED_LIB = libsparsemap.so #CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC -#CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC -CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC +CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -D_FORTIFY_SOURCE=0 -O0 -g -std=c11 -Iinclude/ -fPIC +#CFLAGS = -DREENTRENT_SPARSEMAP -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC -#TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -Itests/ -fPIC +TEST_FLAGS = -DREENTRENT_SPARSEMAP -DDEBUG -Wall -Wextra -Wpedantic -O0 -D_FORTIFY_SOURCE=0 -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -Itests/ -fPIC -TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC +#TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC TESTS = tests/test diff --git a/include/sparsemap.h b/include/sparsemap.h index bf749aa..a08e386 100644 --- a/include/sparsemap.h +++ b/include/sparsemap.h @@ -109,6 +109,9 @@ typedef uint64_t sm_bitvec_t; * @returns The newly allocated sparsemap reference. */ sparsemap_t *sparsemap(size_t size); +#ifdef REENTRENT_SPARSEMAP +sparsemap_t *sparsemap_r(size_t size); +#endif /** @brief Allocate a new, empty sparsemap_t that references (wraps) the buffer * \b data of \b size bytes to use for storage of bitmap data. @@ -190,6 +193,9 @@ void sparsemap_clear(sparsemap_t *map); * supported. */ sparsemap_t *sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data); +#ifdef REENTRENT_SPARSEMAP +sparsemap_t *sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *data); +#endif /** @brief Calculate remaining capacity, approaches 0 when full. * diff --git a/src/sparsemap.c b/src/sparsemap.c index 718c4ef..52575e5 100644 --- a/src/sparsemap.c +++ b/src/sparsemap.c @@ -33,6 +33,9 @@ #include #include #include +#ifdef REENTRENT_SPARSEMAP +#include +#endif #ifdef SPARSEMAP_DIAGNOSTIC #pragma GCC diagnostic push @@ -109,6 +112,7 @@ enum __SM_CHUNK_INFO { }; #define SM_CHUNK_GET_FLAGS(from, at) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK << ((at)*2))) >> ((at)*2) +gsb #define SM_IS_REENTRENT(from) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK) typedef struct { sm_bitvec_t *m_data; @@ -966,6 +970,19 @@ sparsemap(size_t size) return map; } +#ifdef REENTRENT_SPARSEMAP +sparsemap_t * +sparsemap_r(size_t size) +{ + sparsemap_t *map = sparsemap(size + sizeof(pthread_mutex_t)); + sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map + sizeof(pthread_mutex_t)); + pthread_mutex_t *mutex = (pthread_mutex_t *)map; + memcpy(rm, map, size); + pthread_mutex_init(mutex, NULL); + return map; +} +#endif + sparsemap_t * sparsemap_wrap(uint8_t *data, size_t size) { @@ -1008,13 +1025,24 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data) /* Ensure that m_data is 8-byte aligned. */ size_t total_size = sizeof(sparsemap_t) + data_size; +#ifdef REENTRENT_SPARSEMAP + total_size += sizeof(pthread_mutex_t); +#endif size_t padding = total_size % 8 == 0 ? 0 : 8 - (total_size % 8); total_size += padding; +#ifdef REENTRENT_SPARSEMAP + sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map - sizeof(pthread_mutex_t)); + sparsemap_t *m = (sparsemap_t *)realloc(rm, total_size); +#else sparsemap_t *m = (sparsemap_t *)realloc(map, total_size); +#endif if (!m) { return NULL; } +#ifdef REENTRENT_SPARSEMAP + sparsemap_t *m = (sparsemap_t *)((uintptr_t)m + sizeof(pthread_mutex_t)); +#endif memset(((uint8_t *)m) + sizeof(sparsemap_t) + (m->m_capacity * sizeof(uint8_t)), 0, size - m->m_capacity + padding); m->m_capacity = data_size; m->m_data = (uint8_t *)(((uintptr_t)m + sizeof(sparsemap_t)) & ~(uintptr_t)7); @@ -1028,6 +1056,18 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data) } } +#ifdef REENTRENT_SPARSEMAP +sparsemap_t *sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *data) +{ + pthread_mutex_t *mutex = (pthread_mutex_t *)map; + sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map + sizeof(pthread_mutex_t)); + pthread_mutex_lock(mutex); + sparsemap_t *ret = sparsemap_set_data_size(rm size, data); + pthread_mutex_unlock(mutex); + return ret; +} +#endif + double sparsemap_capacity_remaining(sparsemap_t *map) { diff --git a/tests/test.c b/tests/test.c index ffc7138..0c87e85 100644 --- a/tests/test.c +++ b/tests/test.c @@ -38,6 +38,10 @@ struct user_data { int foo; }; +#ifdef REENTRENT_SPARSEMAP +#define sparsemap() GSB +#endif + /* -------------------------- Supporting Functions for Testing */ void -- 2.45.2 From 5157f1ce43a610dc7cc6183822f1ac83860123ad Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 2 May 2024 11:16:20 -0400 Subject: [PATCH 2/2] WIP --- Makefile | 4 ++-- examples/ex_1.c | 3 +++ examples/soak.c | 2 +- include/sparsemap.h | 6 ++--- src/sparsemap.c | 57 ++++++++++++++++++++++++--------------------- tests/test.c | 35 +++++++++++++++++++++++++++- 6 files changed, 74 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 440aff4..03bc544 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,14 @@ SHARED_LIB = libsparsemap.so #CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC -CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -D_FORTIFY_SOURCE=0 -O0 -g -std=c11 -Iinclude/ -fPIC +CFLAGS = -DREENTRENT_SPARSEMAP -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -DREENTRENT_SPARSEMAP -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC -TEST_FLAGS = -DREENTRENT_SPARSEMAP -DDEBUG -Wall -Wextra -Wpedantic -O0 -D_FORTIFY_SOURCE=0 -g -std=c11 -Iinclude/ -Itests/ -fPIC +TEST_FLAGS = -DREENTRENT_SPARSEMAP -DDEBUG -Wall -Wextra -Wpedantic -O0 -Wno-fortify-source -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC diff --git a/examples/ex_1.c b/examples/ex_1.c index dea68f1..2b95971 100644 --- a/examples/ex_1.c +++ b/examples/ex_1.c @@ -17,6 +17,9 @@ /* !!! Duplicated here for testing purposes. Keep in sync, or suffer. !!! */ struct sparsemap { +#ifdef REENTRENT_SPARSEMAP + pthread_mutex_t m_mutex; +#endif size_t m_capacity; size_t m_data_used; uint8_t *m_data; diff --git a/examples/soak.c b/examples/soak.c index 60dd42c..2fe1810 100644 --- a/examples/soak.c +++ b/examples/soak.c @@ -805,7 +805,7 @@ main(void) size_t len = COUNT; // The largest page is at list[1] because this is a reverse sorted list. pgno_t pg = list[0] ? list[1] + 1 : 0; -// if (toss(6) + 1 < 7) { + // if (toss(6) + 1 < 7) { if (true) { // disable shrinking for now... MDB_IDL new_list = mdb_midl_alloc(len); sparsemap_t *new_map = sparsemap(INITIAL_AMOUNT); diff --git a/include/sparsemap.h b/include/sparsemap.h index a08e386..9a90a52 100644 --- a/include/sparsemap.h +++ b/include/sparsemap.h @@ -109,9 +109,6 @@ typedef uint64_t sm_bitvec_t; * @returns The newly allocated sparsemap reference. */ sparsemap_t *sparsemap(size_t size); -#ifdef REENTRENT_SPARSEMAP -sparsemap_t *sparsemap_r(size_t size); -#endif /** @brief Allocate a new, empty sparsemap_t that references (wraps) the buffer * \b data of \b size bytes to use for storage of bitmap data. @@ -210,6 +207,9 @@ sparsemap_t *sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *d * 100.0 when empty */ double sparsemap_capacity_remaining(sparsemap_t *map); +#ifdef REENTRENT_SPARSEMAP +sparsemap_t *sparsemap_capacity_remaining_r(size_t size); +#endif /** @brief Returns the capacity of the underlying byte array in bytes. * diff --git a/src/sparsemap.c b/src/sparsemap.c index 52575e5..d6302bd 100644 --- a/src/sparsemap.c +++ b/src/sparsemap.c @@ -33,6 +33,7 @@ #include #include #include + #ifdef REENTRENT_SPARSEMAP #include #endif @@ -112,13 +113,15 @@ enum __SM_CHUNK_INFO { }; #define SM_CHUNK_GET_FLAGS(from, at) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK << ((at)*2))) >> ((at)*2) -gsb #define SM_IS_REENTRENT(from) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK) typedef struct { sm_bitvec_t *m_data; } __sm_chunk_t; -struct __attribute__((aligned(8))) sparsemap { +struct sparsemap { +#ifdef REENTRENT_SPARSEMAP + pthread_mutex_t m_mutex; +#endif size_t m_capacity; /* The total size of m_data */ size_t m_data_used; /* The used size of m_data */ uint8_t *m_data; /* The serialized bitmap data */ @@ -970,19 +973,6 @@ sparsemap(size_t size) return map; } -#ifdef REENTRENT_SPARSEMAP -sparsemap_t * -sparsemap_r(size_t size) -{ - sparsemap_t *map = sparsemap(size + sizeof(pthread_mutex_t)); - sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map + sizeof(pthread_mutex_t)); - pthread_mutex_t *mutex = (pthread_mutex_t *)map; - memcpy(rm, map, size); - pthread_mutex_init(mutex, NULL); - return map; -} -#endif - sparsemap_t * sparsemap_wrap(uint8_t *data, size_t size) { @@ -999,6 +989,9 @@ sparsemap_init(sparsemap_t *map, uint8_t *data, size_t size) map->m_data = data; map->m_data_used = 0; map->m_capacity = size; +#ifdef REENTRENT_SPARSEMAP + pthread_mutex_init(&map->m_mutex); +#endif sparsemap_clear(map); } @@ -1032,8 +1025,8 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data) total_size += padding; #ifdef REENTRENT_SPARSEMAP - sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map - sizeof(pthread_mutex_t)); - sparsemap_t *m = (sparsemap_t *)realloc(rm, total_size); + sparsemap_t *mem = (sparsemap_t *)((uintptr_t)map - sizeof(pthread_mutex_t)); + sparsemap_t *m = (sparsemap_t *)realloc(mem, total_size); #else sparsemap_t *m = (sparsemap_t *)realloc(map, total_size); #endif @@ -1041,12 +1034,13 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data) return NULL; } #ifdef REENTRENT_SPARSEMAP - sparsemap_t *m = (sparsemap_t *)((uintptr_t)m + sizeof(pthread_mutex_t)); + m = (sparsemap_t *)((uintptr_t)m + sizeof(pthread_mutex_t)); #endif memset(((uint8_t *)m) + sizeof(sparsemap_t) + (m->m_capacity * sizeof(uint8_t)), 0, size - m->m_capacity + padding); m->m_capacity = data_size; m->m_data = (uint8_t *)(((uintptr_t)m + sizeof(sparsemap_t)) & ~(uintptr_t)7); - __sm_when_diag({ __sm_assert(IS_8_BYTE_ALIGNED(m->m_data)); }) return m; + __sm_when_diag({ __sm_assert(IS_8_BYTE_ALIGNED(m->m_data)); }); + return m; } else { if (data != NULL && data_size > sparsemap_get_capacity(map) && data != map->m_data) { map->m_data = data; @@ -1057,14 +1051,25 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data) } #ifdef REENTRENT_SPARSEMAP -sparsemap_t *sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *data) +sparsemap_t * +sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *data) { - pthread_mutex_t *mutex = (pthread_mutex_t *)map; - sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map + sizeof(pthread_mutex_t)); - pthread_mutex_lock(mutex); - sparsemap_t *ret = sparsemap_set_data_size(rm size, data); - pthread_mutex_unlock(mutex); - return ret; + sparsemap_t *retval; + pthread_mutex_t *mutex; + + if (SM_IS_REENTRENT_MAP(map)) { + mutex = SM_REENTRENT_MUTEX(map); + pthread_mutex_lock(mutex); + sparsemap_t *real_map = SM_REENTRENT_MAP(map); + retval = sparsemap_set_data_size(real_map, size, data); + /* realloc() could have moved the map, so get these again. */ + mutex = SM_REENTRENT_MUTEX(retval); + retval = SM_REENTRENT_MAP(retval); + pthread_mutex_unlock(mutex); + } else { + retval = sparsemap_set_data_size(map, size, data); + } + return retval; } #endif diff --git a/tests/test.c b/tests/test.c index 0c87e85..bd95675 100644 --- a/tests/test.c +++ b/tests/test.c @@ -29,6 +29,9 @@ /* !!! Duplicated here for testing purposes. Keep in sync, or suffer. !!! */ struct sparsemap { +#ifdef REENTRENT_SPARSEMAP + pthread_mutex_t m_mutex; +#endif size_t m_capacity; size_t m_data_used; uint8_t *m_data; @@ -39,7 +42,8 @@ struct user_data { }; #ifdef REENTRENT_SPARSEMAP -#define sparsemap() GSB +#define sparsemap(size) sparsemap_r(size) +#define sparsemap_set_data_size(map, size, data) sparsemap_set_data_size_r(map, size, data) #endif /* -------------------------- Supporting Functions for Testing */ @@ -1011,6 +1015,34 @@ test_api_span(const MunitParameter params[], void *data) return MUNIT_OK; } +static void * +test_api_reentrent_setup(const MunitParameter params[], void *user_data) +{ + sparsemap_t *map = sparsemap_r(1024); + assert_ptr_not_null(map); + return (void *)map; +} +static void +test_api_reentrent_tear_down(void *fixture) +{ + sparsemap_t *map = (sparsemap_t *)fixture; + free(map); + test_api_tear_down(fixture); +} +static MunitResult +test_api_reentrent(const MunitParameter params[], void *data) +{ + sparsemap_t *map = (sparsemap_t *)data; + (void)params; + + assert_ptr_not_null(map); + sparsemap_set_data_size_r(map, 2048, NULL); + + // TODO... moar. + + return MUNIT_OK; +} + // clang-format off static MunitTest api_test_suite[] = { { (char *)"/new", test_api_new, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }, @@ -1042,6 +1074,7 @@ static MunitTest api_test_suite[] = { { (char *)"/rank/true", test_api_rank_true, test_api_rank_true_setup, test_api_rank_true_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, { (char *)"/rank/false", test_api_rank_false, test_api_rank_false_setup, test_api_rank_false_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, { (char *)"/span", test_api_span, test_api_span_setup, test_api_span_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, + { (char *)"/reentrent", test_api_reentrent, test_api_reentrent_setup, test_api_reentrent_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, { NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL } }; // clang-format on -- 2.45.2