greg/sparsemap
1
0
Fork 0
Code Issues Pull requests 1 Packages Projects Releases Wiki Activity

fix scan

Browse source
This commit is contained in:
Gregory Burd 2024-05-02 14:55:04 -04:00
parent 6b6ed5290f
commit c9ae042b40
4 changed files with 63 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
examples/soak.c
View file

@ -533,6 +533,21 @@ verify_empty_sparsemap(sparsemap_t *map, pgno_t pg, unsigned len)
return true;
}
bool
verify_sm_is_first_available_span(sparsemap_t *map, sparsemap_idx_t idx, size_t len, bool value)
{
for (sparsemap_idx_t i = 0; i < idx + len; i++) {
sparsemap_idx_t j = 0;
while (sparsemap_is_set(map, i + j) == value && j < len && j < idx + len) {
j++;
}
if (j == len) {
return i == idx;
}
}
return false;
}
bool
verify_sm_eq_ml(sparsemap_t *map, MDB_IDL list)
{
@ -702,6 +717,7 @@ main(void)
sl = pgno;
e = nsts();
td_add(b_span_loc, e - b, 1);
assert(verify_sm_is_first_available_span(map, pgno, n, true));
}
assert(verify_span_midl(list, sl, n));
assert(verify_span_sparsemap(map, sl, n));
@ -761,24 +777,25 @@ main(void)
// Once we've used half of the free list, let's replenish it a bit.
if (list[0] < amt / 2) {
do {
pgno_t pg;
pgno_t pgno;
size_t len, retries = amt;
do {
len = toss(15) + 1;
pg = sparsemap_span(map, 0, len, false);
pgno = sparsemap_span(map, 0, len, false);
assert(verify_sm_is_first_available_span(map, pgno, n, false));
//__diag("%zu\t%zu,%zu\n", iterations, replenish, retries);
} while (SPARSEMAP_NOT_FOUND(pg) && --retries);
} while (SPARSEMAP_NOT_FOUND(pgno) && --retries);
if (retries == 0) {
goto larger_please;
}
if (SPARSEMAP_FOUND(pg)) {
assert(verify_empty_midl(list, pg, len));
assert(verify_empty_sparsemap(map, pg, len));
if (SPARSEMAP_FOUND(pgno)) {
assert(verify_empty_midl(list, pgno, len));
assert(verify_empty_sparsemap(map, pgno, len));
assert(verify_sm_eq_ml(map, list));
if (list[-1] - list[0] < len) {
mdb_midl_need(&list, list[-1] + len);
}
for (size_t i = pg; i < pg + len; i++) {
for (size_t i = pgno; i < pgno + len; i++) {
assert(verify_midl_contains(list, i) == false);
assert(sparsemap_is_set(map, i) == false);
mdb_midl_insert(list, i);
@ -788,8 +805,8 @@ main(void)
}
mdb_midl_sort(list);
assert(verify_midl_nodups(list));
assert(verify_span_midl(list, pg, len));
assert(verify_span_sparsemap(map, pg, len));
assert(verify_span_midl(list, pgno, len));
assert(verify_span_sparsemap(map, pgno, len));
}
assert(verify_sm_eq_ml(map, list));
replenish++;

12
include/sparsemap.h
View file

@ -249,16 +249,14 @@ size_t sparsemap_get_size(sparsemap_t *map);
/** @brief Provides a method for a callback function to examine every bit set in
* the index.
*
* This decompresses the whole bitmap and invokes #scanner() passing a 64bit
* "vector" of bits in order from 0 index to the end of the map. Using standard
* bit masking techniques it is possible to read each bit from LSB to MSB in
* these vectors to read the entire content of the bitmap index (see
* examples/ex_4.c).
* This decompresses the whole bitmap and invokes #scanner() passing an array
* of the positions of set bits in order from 0 index to the end of the map.
*
* @param[in] map The sparsemap reference.
* @param[in] skip Start the scan after "skip" bits.
* @param[in] skip Start the scan after \b skip position in the map.
* @param[in] aux Auxiliary information passed to the scanner.
*/
void sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t vec[], size_t n), size_t skip);
void sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t vec[], size_t n, void *aux), size_t skip, void *aux);
/** @brief Merges the values from \b other into the \b map, \b other is unchanged.
* \b other bitmap while removing them from \b map.

40
src/sparsemap.c
View file

@ -680,24 +680,27 @@ __sm_chunk_map_rank(__sm_chunk_t *map, size_t *offset, size_t idx, size_t *pos,
* @param[in] map The chunk in question.
* @param[in] start
* @param[in] scanner
* @param[in] skip The number of
* @returns the number of (set) bits that were passed to the scanner
*/
static size_t
__sm_chunk_map_scan(__sm_chunk_t *map, sm_idx_t start, void (*scanner)(sm_idx_t[], size_t), size_t skip)
__sm_chunk_map_scan(__sm_chunk_t *map, sm_idx_t start, void (*scanner)(sm_idx_t[], size_t, void *aux), size_t skip, void *aux)
{
size_t ret = 0;
register uint8_t *p = (uint8_t *)map->m_data;
sm_idx_t buffer[SM_BITS_PER_VECTOR];
for (size_t i = 0; i < sizeof(sm_bitvec_t); i++, p++) {
if (*p == 0) {
/* skip the zeroes */
/* Skip chunks that are all zeroes. */
skip -= skip > SM_BITS_PER_VECTOR ? SM_BITS_PER_VECTOR : skip;
continue;
}
for (int j = 0; j < SM_FLAGS_PER_INDEX_BYTE; j++) {
size_t flags = SM_CHUNK_GET_FLAGS(*p, j);
if (flags == SM_PAYLOAD_NONE || flags == SM_PAYLOAD_ZEROS) {
/* ignore the zeroes */
/* Skip when all zeroes. */
skip -= skip > SM_BITS_PER_VECTOR ? SM_BITS_PER_VECTOR : skip;
} else if (flags == SM_PAYLOAD_ONES) {
if (skip) {
if (skip >= SM_BITS_PER_VECTOR) {
@ -709,26 +712,31 @@ __sm_chunk_map_scan(__sm_chunk_t *map, sm_idx_t start, void (*scanner)(sm_idx_t[
for (size_t b = skip; b < SM_BITS_PER_VECTOR; b++) {
buffer[n++] = start + b;
}
scanner(&buffer[0], n);
scanner(&buffer[0], n, aux);
ret += n;
skip = 0;
} else {
for (size_t b = 0; b < SM_BITS_PER_VECTOR; b++) {
buffer[b] = start + b;
}
scanner(&buffer[0], SM_BITS_PER_VECTOR);
scanner(&buffer[0], SM_BITS_PER_VECTOR, aux);
ret += SM_BITS_PER_VECTOR;
}
} else if (flags == SM_PAYLOAD_MIXED) {
sm_bitvec_t w = map->m_data[1 + __sm_chunk_map_get_position(map, i * SM_FLAGS_PER_INDEX_BYTE + j)];
int n = 0;
size_t n = 0;
if (skip) {
if (skip >= SM_BITS_PER_VECTOR) {
skip -= SM_BITS_PER_VECTOR;
ret += SM_BITS_PER_VECTOR;
continue;
}
for (int b = 0; b < SM_BITS_PER_VECTOR; b++) {
if (w & ((sm_bitvec_t)1 << b)) {
if (skip > 0) {
skip--;
continue;
// TODO: unreachable lines below... why?
}
if (w & ((sm_bitvec_t)1 << b)) {
buffer[n++] = start + b;
ret++;
}
@ -742,7 +750,7 @@ __sm_chunk_map_scan(__sm_chunk_t *map, sm_idx_t start, void (*scanner)(sm_idx_t[
ret += n;
}
__sm_assert(n > 0);
scanner(&buffer[0], n);
scanner(&buffer[0], n, aux);
}
}
}
@ -993,10 +1001,6 @@ sparsemap_open(sparsemap_t *map, uint8_t *data, size_t size)
map->m_capacity = size;
}
/*
* TODO/NOTE: This is a dangerous operation because we cannot verify that
* data_size is not exceeding the size of the underlying buffer.
*/
sparsemap_t *
sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data)
{
@ -1020,7 +1024,9 @@ sparsemap_set_data_size(sparsemap_t *map, size_t size, uint8_t *data)
m->m_data = (uint8_t *)(((uintptr_t)m + sizeof(sparsemap_t)) & ~(uintptr_t)7);
__sm_when_diag({ __sm_assert(IS_8_BYTE_ALIGNED(m->m_data)); }) return m;
} else {
if (data != NULL && data_size > sparsemap_get_capacity(map) && data != map->m_data) {
/* NOTE: It is up to the caller to realloc their buffer and provide it here
for reassignment. */
if (data != NULL && data_size > sparsemap_get_capacity(map) && data != map->m_data) {
map->m_data = data;
}
map->m_capacity = size;
@ -1240,7 +1246,7 @@ sparsemap_get_size(sparsemap_t *map)
* Decompresses the whole bitmap; calls scanner for all bits.
*/
void
sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t[], size_t), size_t skip)
sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t[], size_t, void *aux), size_t skip, void *aux)
{
uint8_t *p = __sm_get_chunk_map_data(map, 0);
size_t count = __sm_get_chunk_map_count(map);
@ -1250,7 +1256,7 @@ sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t[], size_t), size_t ski
p += sizeof(sm_idx_t);
__sm_chunk_t chunk;
__sm_chunk_map_init(&chunk, p);
size_t skipped = __sm_chunk_map_scan(&chunk, start, scanner, skip);
size_t skipped = __sm_chunk_map_scan(&chunk, start, scanner, skip, aux);
if (skip) {
assert(skip >= skipped);
skip -= skipped;

20
tests/test.c
View file

@ -524,10 +524,7 @@ test_api_scan_setup(const MunitParameter params[], void *user_data)
uint8_t *buf = munit_calloc(1024, sizeof(uint8_t));
assert_ptr_not_null(buf);
sparsemap_t *map = (sparsemap_t *)test_api_setup(params, user_data);
sparsemap_init(map, buf, 1024);
sm_bitmap_from_uint64(map, 0, ((uint64_t)0xfeedface << 32) | 0xbadc0ffee);
return (void *)map;
}
static void
@ -539,11 +536,14 @@ test_api_scan_tear_down(void *fixture)
test_api_tear_down(fixture);
}
void
scan_for_0xfeedfacebadcoffee(sm_idx_t v[], size_t n)
scan_for_0xfeedfacebadcoffee(sm_idx_t v[], size_t n, void *aux)
{
/* Called multiple times */
((void)v);
((void)n);
size_t bit_pos[] = {1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 22, 23, 24, 26, 27, 29, 31, 32, 33, 34, 35, 38, 39, 41, 43, 44, 45, 46, 47, 48, 50, 51, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63};
(void)aux;
for (size_t i = 0; i < n; i++) {
assert(v[i] == bit_pos[i]);
}
}
static MunitResult
test_api_scan(const MunitParameter params[], void *data)
@ -552,10 +552,8 @@ test_api_scan(const MunitParameter params[], void *data)
(void)params;
assert_ptr_not_null(map);
sparsemap_set(map, 4200, true);
assert_true(sparsemap_is_set(map, 4200));
sparsemap_scan(map, scan_for_0xfeedfacebadcoffee, 0);
sm_bitmap_from_uint64(map, 0, ((uint64_t)0xfeedface << 32) | 0xbadc0ffee);
sparsemap_scan(map, scan_for_0xfeedfacebadcoffee, 0, NULL);
return MUNIT_OK;
}