WIP: gburd/thread-safe #7

Draft
greg wants to merge 2 commits from gburd/thread-safe into main
6 changed files with 74 additions and 33 deletions
Showing only changes of commit 5157f1ce43 - Show all commits

View file

@ -5,14 +5,14 @@ SHARED_LIB = libsparsemap.so
#CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -D_FORTIFY_SOURCE=0 -O0 -g -std=c11 -Iinclude/ -fPIC CFLAGS = -DREENTRENT_SPARSEMAP -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -DREENTRENT_SPARSEMAP -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -DREENTRENT_SPARSEMAP -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC
TEST_FLAGS = -DREENTRENT_SPARSEMAP -DDEBUG -Wall -Wextra -Wpedantic -O0 -D_FORTIFY_SOURCE=0 -g -std=c11 -Iinclude/ -Itests/ -fPIC TEST_FLAGS = -DREENTRENT_SPARSEMAP -DDEBUG -Wall -Wextra -Wpedantic -O0 -Wno-fortify-source -g -std=c11 -Iinclude/ -Itests/ -fPIC
#TEST_FLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -Itests/ -fPIC
#TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC #TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC
#TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC #TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC

View file

@ -17,6 +17,9 @@
/* !!! Duplicated here for testing purposes. Keep in sync, or suffer. !!! */ /* !!! Duplicated here for testing purposes. Keep in sync, or suffer. !!! */
struct sparsemap { struct sparsemap {
#ifdef REENTRENT_SPARSEMAP
pthread_mutex_t m_mutex;
#endif
size_t m_capacity; size_t m_capacity;
size_t m_data_used; size_t m_data_used;
uint8_t *m_data; uint8_t *m_data;

View file

@ -805,7 +805,7 @@ main(void)
size_t len = COUNT; size_t len = COUNT;
// The largest page is at list[1] because this is a reverse sorted list. // The largest page is at list[1] because this is a reverse sorted list.
pgno_t pg = list[0] ? list[1] + 1 : 0; pgno_t pg = list[0] ? list[1] + 1 : 0;
// if (toss(6) + 1 < 7) { // if (toss(6) + 1 < 7) {
if (true) { // disable shrinking for now... if (true) { // disable shrinking for now...
MDB_IDL new_list = mdb_midl_alloc(len); MDB_IDL new_list = mdb_midl_alloc(len);
sparsemap_t *new_map = sparsemap(INITIAL_AMOUNT); sparsemap_t *new_map = sparsemap(INITIAL_AMOUNT);

View file

@ -109,9 +109,6 @@ typedef uint64_t sm_bitvec_t;
* @returns The newly allocated sparsemap reference. * @returns The newly allocated sparsemap reference.
*/ */
sparsemap_t *sparsemap(size_t size); sparsemap_t *sparsemap(size_t size);
#ifdef REENTRENT_SPARSEMAP
sparsemap_t *sparsemap_r(size_t size);
#endif
/** @brief Allocate a new, empty sparsemap_t that references (wraps) the buffer /** @brief Allocate a new, empty sparsemap_t that references (wraps) the buffer
* \b data of \b size bytes to use for storage of bitmap data. * \b data of \b size bytes to use for storage of bitmap data.
@ -210,6 +207,9 @@ sparsemap_t *sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *d
* 100.0 when empty * 100.0 when empty
*/ */
double sparsemap_capacity_remaining(sparsemap_t *map); double sparsemap_capacity_remaining(sparsemap_t *map);
#ifdef REENTRENT_SPARSEMAP
sparsemap_t *sparsemap_capacity_remaining_r(size_t size);
#endif
/** @brief Returns the capacity of the underlying byte array in bytes. /** @brief Returns the capacity of the underlying byte array in bytes.
* *

View file

@ -33,6 +33,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#ifdef REENTRENT_SPARSEMAP #ifdef REENTRENT_SPARSEMAP
#include <pthread.h> #include <pthread.h>
#endif #endif
@ -112,13 +113,15 @@ enum __SM_CHUNK_INFO {
}; };
#define SM_CHUNK_GET_FLAGS(from, at) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK << ((at)*2))) >> ((at)*2) #define SM_CHUNK_GET_FLAGS(from, at) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK << ((at)*2))) >> ((at)*2)
gsb #define SM_IS_REENTRENT(from) (((from)) & ((sm_bitvec_t)SM_FLAG_MASK)
typedef struct { typedef struct {
sm_bitvec_t *m_data; sm_bitvec_t *m_data;
} __sm_chunk_t; } __sm_chunk_t;
struct __attribute__((aligned(8))) sparsemap { struct sparsemap {
#ifdef REENTRENT_SPARSEMAP
pthread_mutex_t m_mutex;
#endif
size_t m_capacity; /* The total size of m_data */ size_t m_capacity; /* The total size of m_data */
size_t m_data_used; /* The used size of m_data */ size_t m_data_used; /* The used size of m_data */
uint8_t *m_data; /* The serialized bitmap data */ uint8_t *m_data; /* The serialized bitmap data */
@ -970,19 +973,6 @@ sparsemap(size_t size)
return map; return map;
} }
#ifdef REENTRENT_SPARSEMAP
sparsemap_t *
sparsemap_r(size_t size)
{
sparsemap_t *map = sparsemap(size + sizeof(pthread_mutex_t));
sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map + sizeof(pthread_mutex_t));
pthread_mutex_t *mutex = (pthread_mutex_t *)map;
memcpy(rm, map, size);
pthread_mutex_init(mutex, NULL);
return map;
}
#endif
sparsemap_t * sparsemap_t *
sparsemap_wrap(uint8_t *data, size_t size) sparsemap_wrap(uint8_t *data, size_t size)
{ {
@ -999,6 +989,9 @@ sparsemap_init(sparsemap_t *map, uint8_t *data, size_t size)
map->m_data = data; map->m_data = data;
map->m_data_used = 0; map->m_data_used = 0;
map->m_capacity = size; map->m_capacity = size;
#ifdef REENTRENT_SPARSEMAP
pthread_mutex_init(&map->m_mutex);
#endif
sparsemap_clear(map); sparsemap_clear(map);
} }
@ -1032,8 +1025,8 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data)
total_size += padding; total_size += padding;
#ifdef REENTRENT_SPARSEMAP #ifdef REENTRENT_SPARSEMAP
sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map - sizeof(pthread_mutex_t)); sparsemap_t *mem = (sparsemap_t *)((uintptr_t)map - sizeof(pthread_mutex_t));
sparsemap_t *m = (sparsemap_t *)realloc(rm, total_size); sparsemap_t *m = (sparsemap_t *)realloc(mem, total_size);
#else #else
sparsemap_t *m = (sparsemap_t *)realloc(map, total_size); sparsemap_t *m = (sparsemap_t *)realloc(map, total_size);
#endif #endif
@ -1041,12 +1034,13 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data)
return NULL; return NULL;
} }
#ifdef REENTRENT_SPARSEMAP #ifdef REENTRENT_SPARSEMAP
sparsemap_t *m = (sparsemap_t *)((uintptr_t)m + sizeof(pthread_mutex_t)); m = (sparsemap_t *)((uintptr_t)m + sizeof(pthread_mutex_t));
#endif #endif
memset(((uint8_t *)m) + sizeof(sparsemap_t) + (m->m_capacity * sizeof(uint8_t)), 0, size - m->m_capacity + padding); memset(((uint8_t *)m) + sizeof(sparsemap_t) + (m->m_capacity * sizeof(uint8_t)), 0, size - m->m_capacity + padding);
m->m_capacity = data_size; m->m_capacity = data_size;
m->m_data = (uint8_t *)(((uintptr_t)m + sizeof(sparsemap_t)) & ~(uintptr_t)7); m->m_data = (uint8_t *)(((uintptr_t)m + sizeof(sparsemap_t)) & ~(uintptr_t)7);
__sm_when_diag({ __sm_assert(IS_8_BYTE_ALIGNED(m->m_data)); }) return m; __sm_when_diag({ __sm_assert(IS_8_BYTE_ALIGNED(m->m_data)); });
return m;
} else { } else {
if (data != NULL && data_size > sparsemap_get_capacity(map) && data != map->m_data) { if (data != NULL && data_size > sparsemap_get_capacity(map) && data != map->m_data) {
map->m_data = data; map->m_data = data;
@ -1057,14 +1051,25 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data)
} }
#ifdef REENTRENT_SPARSEMAP #ifdef REENTRENT_SPARSEMAP
sparsemap_t *sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *data) sparsemap_t *
sparsemap_set_data_size_r(sparsemap_t *map, size_t size, uint8_t *data)
{ {
pthread_mutex_t *mutex = (pthread_mutex_t *)map; sparsemap_t *retval;
sparsemap_t *rm = (sparsemap_t *)((uintptr_t)map + sizeof(pthread_mutex_t)); pthread_mutex_t *mutex;
pthread_mutex_lock(mutex);
sparsemap_t *ret = sparsemap_set_data_size(rm size, data); if (SM_IS_REENTRENT_MAP(map)) {
pthread_mutex_unlock(mutex); mutex = SM_REENTRENT_MUTEX(map);
return ret; pthread_mutex_lock(mutex);
sparsemap_t *real_map = SM_REENTRENT_MAP(map);
retval = sparsemap_set_data_size(real_map, size, data);
/* realloc() could have moved the map, so get these again. */
mutex = SM_REENTRENT_MUTEX(retval);
retval = SM_REENTRENT_MAP(retval);
pthread_mutex_unlock(mutex);
} else {
retval = sparsemap_set_data_size(map, size, data);
}
return retval;
} }
#endif #endif

View file

@ -29,6 +29,9 @@
/* !!! Duplicated here for testing purposes. Keep in sync, or suffer. !!! */ /* !!! Duplicated here for testing purposes. Keep in sync, or suffer. !!! */
struct sparsemap { struct sparsemap {
#ifdef REENTRENT_SPARSEMAP
pthread_mutex_t m_mutex;
#endif
size_t m_capacity; size_t m_capacity;
size_t m_data_used; size_t m_data_used;
uint8_t *m_data; uint8_t *m_data;
@ -39,7 +42,8 @@ struct user_data {
}; };
#ifdef REENTRENT_SPARSEMAP #ifdef REENTRENT_SPARSEMAP
#define sparsemap() GSB #define sparsemap(size) sparsemap_r(size)
#define sparsemap_set_data_size(map, size, data) sparsemap_set_data_size_r(map, size, data)
#endif #endif
/* -------------------------- Supporting Functions for Testing */ /* -------------------------- Supporting Functions for Testing */
@ -1011,6 +1015,34 @@ test_api_span(const MunitParameter params[], void *data)
return MUNIT_OK; return MUNIT_OK;
} }
static void *
test_api_reentrent_setup(const MunitParameter params[], void *user_data)
{
sparsemap_t *map = sparsemap_r(1024);
assert_ptr_not_null(map);
return (void *)map;
}
static void
test_api_reentrent_tear_down(void *fixture)
{
sparsemap_t *map = (sparsemap_t *)fixture;
free(map);
test_api_tear_down(fixture);
}
static MunitResult
test_api_reentrent(const MunitParameter params[], void *data)
{
sparsemap_t *map = (sparsemap_t *)data;
(void)params;
assert_ptr_not_null(map);
sparsemap_set_data_size_r(map, 2048, NULL);
// TODO... moar.
return MUNIT_OK;
}
// clang-format off // clang-format off
static MunitTest api_test_suite[] = { static MunitTest api_test_suite[] = {
{ (char *)"/new", test_api_new, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }, { (char *)"/new", test_api_new, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
@ -1042,6 +1074,7 @@ static MunitTest api_test_suite[] = {
{ (char *)"/rank/true", test_api_rank_true, test_api_rank_true_setup, test_api_rank_true_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, { (char *)"/rank/true", test_api_rank_true, test_api_rank_true_setup, test_api_rank_true_tear_down, MUNIT_TEST_OPTION_NONE, NULL },
{ (char *)"/rank/false", test_api_rank_false, test_api_rank_false_setup, test_api_rank_false_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, { (char *)"/rank/false", test_api_rank_false, test_api_rank_false_setup, test_api_rank_false_tear_down, MUNIT_TEST_OPTION_NONE, NULL },
{ (char *)"/span", test_api_span, test_api_span_setup, test_api_span_tear_down, MUNIT_TEST_OPTION_NONE, NULL }, { (char *)"/span", test_api_span, test_api_span_setup, test_api_span_tear_down, MUNIT_TEST_OPTION_NONE, NULL },
{ (char *)"/reentrent", test_api_reentrent, test_api_reentrent_setup, test_api_reentrent_tear_down, MUNIT_TEST_OPTION_NONE, NULL },
{ NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL } { NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }
}; };
// clang-format on // clang-format on