diff --git a/examples/ex_2.c b/examples/ex_2.c index d2250b6..d897be4 100644 --- a/examples/ex_2.c +++ b/examples/ex_2.c @@ -38,7 +38,7 @@ main(void) } } // On 1024 KiB of buffer with every other bit set the map holds 7744 bits - // and then runs out of space. This next _set() call will fail/abort. + // and then runs out of space. This next _set() call will fail. sparsemap_set(map, ++i, true); assert(sparsemap_is_set(map, i) == true); return 0; diff --git a/examples/soak.c b/examples/soak.c index 6827b85..b90ad58 100644 --- a/examples/soak.c +++ b/examples/soak.c @@ -75,6 +75,13 @@ void mdb_midl_free(MDB_IDL ids); */ void mdb_midl_shrink(MDB_IDL *idp); +/** Shrink an IDL to a specific size. + * Resize the IDL to \b size if it is larger. + * @param[in,out] idp Address of the IDL to shrink. + * @param[in] size Capacity to have once resized. + */ +void mdb_midl_shrink(MDB_IDL *idp); + /** Make room for num additional elements in an IDL. * @param[in,out] idp Address of the IDL. * @param[in] num Number of elements to make room for. @@ -218,6 +225,17 @@ mdb_midl_shrink(MDB_IDL *idp) } } +void +mdb_midl_shrink_to(MDB_IDL *idp, size_t size) +{ + MDB_IDL ids = *idp; + if (*(--ids) > size && (ids = realloc(ids, (size + 2) * sizeof(MDB_ID)))) { + *ids++ = size; + *idp = ids; + *idp[0] = *idp[0] > size ? size : *idp[0]; + } +} + static int mdb_midl_grow(MDB_IDL *idp, int num) { @@ -426,8 +444,8 @@ toss(size_t max) bool verify_midl_contains(MDB_IDL list, pgno_t pg) { - unsigned index = mdb_midl_search(list, pg); - return index <= list[0] && list[index] == pg; + unsigned idx = mdb_midl_search(list, pg); + return idx <= list[0] && list[idx] == pg; } bool @@ -445,8 +463,8 @@ verify_midl_nodups(MDB_IDL list) bool verify_span_midl(MDB_IDL list, pgno_t pg, unsigned len) { - pgno_t f = mdb_midl_search(list, pg); - bool found = (list[f] == pg) && (f <= list[0]); + pgno_t idx = mdb_midl_search(list, pg); + bool found = idx <= list[0] && list[idx] == pg; if (!found) return false; if (len == 1) @@ -460,8 +478,8 @@ bool verify_empty_midl(MDB_IDL list, pgno_t pg, unsigned len) { for (pgno_t i = pg; i < pg + len; i++) { - pgno_t f = mdb_midl_search(list, pg); - bool found = list[f] == pg && f <= list[0]; + pgno_t idx = mdb_midl_search(list, pg); + bool found = idx <= list[0] && list[idx] == pg; if (found) return false; } @@ -493,22 +511,41 @@ verify_empty_sparsemap(sparsemap_t *map, pgno_t pg, unsigned len) bool verify_sm_eq_ml(sparsemap_t *map, MDB_IDL list) { - for (unsigned i = 0; i <= list[1]; i++) { - pgno_t f = mdb_midl_search(list, i); - bool found = list[f] == i && f <= list[0]; - if (sparsemap_is_set(map, i) != found) + for (int i = 1; i <= list[0]; i++) { + pgno_t pg = list[i]; + unsigned skipped = i == 1 ? 0 : list[i-1] - list[i] - 1; + for (int j = 0; j < skipped; j++) { + if (sparsemap_is_set(map, pg - j) != false) + return false; + } + if (sparsemap_is_set(map, pg) != true) return false; } return true; } void -print_sizes(sparsemap_t *map, MDB_IDL list) +stats(size_t iterations, sparsemap_t *map, MDB_IDL list) { char m[1024], l[1024]; - __diag("idl: %s bytes\tsm: %s bytes\n", bytes_as(MDB_IDL_SIZEOF(list), m, 1024), bytes_as(sparsemap_get_capacity(map), l, 1024)); + __diag("%zu\tidl[%zu/%zu]: %s\tsm: %s\n", iterations, list[-1], list[0], bytes_as(MDB_IDL_SIZEOF(list), m, 1024), bytes_as(sparsemap_get_capacity(map), l, 1024)); } +sparsemap_idx_t +_sparsemap_set(sparsemap_t **map, sparsemap_idx_t idx, bool value) +{ + sparsemap_idx_t l = sparsemap_set(*map, idx, value); + if (errno == ENOSPC) { + *map = sparsemap_set_data_size(*map, sparsemap_get_capacity(*map) + 64, NULL); + assert(*map != NULL); + errno = 0; + } + return l; +} + + +#define INITIAL_AMOUNT 1024 * 2 + /* * A "soak test" that tries to replicate behavior in LMDB for page allocation. */ @@ -524,7 +561,7 @@ main() __diag("starting...\n"); - size_t amt = 1024 * 2; // 1024 * 1024 * 2; + size_t amt = INITIAL_AMOUNT; MDB_IDL list = mdb_midl_alloc(amt); sparsemap_t *map = sparsemap(3 * 1024); @@ -533,18 +570,20 @@ main() // - Sparsemap will compress the set bits using less memory mdb_midl_need(&list, amt); for (size_t pg = 0; pg < amt; pg++) { - mdb_midl_xappend(list, pg); // listed page ids are free - sparsemap_set(map, pg, true); // true means free in our bitmap + // We list every free (unallocated) page in the IDL, while... + mdb_midl_xappend(list, pg); + // ... true (unset in the bitmap) indicates free in the bitmap. + assert(_sparsemap_set(&map, pg, true) == pg); } mdb_midl_sort(list); - print_sizes(map, list); + stats(0, map, list); assert(verify_sm_eq_ml(map, list)); while (1) { unsigned mi; pgno_t ml = 0, sl = 0; - // get an amount [1, 16] of pages to find prefering smaller sizes + // get an amount [1, 16] of pages to find preferring smaller sizes unsigned n = toss(15) + 1; // find a set of pages using the MDB_IDL @@ -610,11 +649,11 @@ main() // acquire the set of pages within the sparsemap if (prefer_mdb_idl_location) { for (pgno_t i = ml; i < ml + n; i++) { - sparsemap_set(map, i, false); + assert(_sparsemap_set(&map, i, false) == i); } } else { for (pgno_t i = sl; i <= sl + n; i++) { - sparsemap_set(map, i, false); + assert(_sparsemap_set(&map, i, false) == i); } } @@ -632,12 +671,12 @@ main() if (SPARSEMAP_FOUND(pg)) { assert(verify_empty_midl(list, pg, len)); assert(verify_empty_sparsemap(map, pg, len)); + if (list[-1] - list[0] < len) + mdb_midl_need(&list, list[-1] + len); for (int i = pg; i < pg + len; i++) { - if (pg + len > list[-1]) - mdb_midl_need(&list, pg + len); assert(verify_midl_contains(list, i) == false); mdb_midl_insert(list, i); - sparsemap_set(map, i, true); + assert(_sparsemap_set(&map, i, true) == i); } mdb_midl_sort(list); assert(verify_midl_nodups(list)); @@ -646,7 +685,41 @@ main() } } while (list[0] < amt - 32); } - print_sizes(map, list); + stats(iterations, map, list); + + // every 100 iterations, either ... + if (iterations % 100 == 0) { + const int COUNT = 1024; + if (toss(6) + 1 < 7) { + // ... add a MiB of 4KiB pages, or + int len = COUNT; + // The largest page is at list[1] because this is a reverse sorted list. + int pg = list[1] + 1; + if (list[0] + COUNT > list[-1]) + mdb_midl_grow(&list, list[0] + len); + for (int i = pg; i < pg + len; i++) { + assert(verify_midl_contains(list, i) == false); + assert(sparsemap_is_set(map, i) == false); + mdb_midl_insert(list, i); + assert(_sparsemap_set(&map, i, true) == i); + } + mdb_midl_sort(list); + assert(verify_midl_nodups(list)); + verify_sm_eq_ml(map, list); + } else { + if (list[-1] > INITIAL_AMOUNT) { + // ... a fraction of the time, remove a MiB of 4KiB pages. + for (int i = 0; i < COUNT; i++) { + pgno_t pg = list[list[0] - i]; + assert(sparsemap_is_set(map, pg) == true); + assert(_sparsemap_set(&map, pg, false) == pg) ; + } + mdb_midl_shrink_to(&list, list[0] - COUNT); + assert(verify_midl_nodups(list)); + verify_sm_eq_ml(map, list); + } + } + } iterations++; } diff --git a/src/sparsemap.c b/src/sparsemap.c index bc97738..109ae65 100644 --- a/src/sparsemap.c +++ b/src/sparsemap.c @@ -1338,7 +1338,7 @@ sparsemap_select(sparsemap_t *map, sparsemap_idx_t n, bool value) if (value) { return SPARSEMAP_IDX_MAX; } else { - return count * SM_CHUNK_MAX_CAPACITY + 1; + return count * SM_CHUNK_MAX_CAPACITY; } } else { // TODO... sparsemap_select(map, -n, value); seek from end, not start