greg/sparsemap
1
0
Fork 0
Code Issues Pull requests 1 Packages Projects Releases Wiki Activity

cleanup

Browse source
This commit is contained in:
Gregory Burd 2024-05-02 21:13:17 -04:00
parent 86798b32bd
commit a7754b05ba
6 changed files with 24 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Makefile
View file

@ -6,9 +6,9 @@ SHARED_LIB = libsparsemap.so
#CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC
CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC
#CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC
#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC
@ -37,6 +37,9 @@ $(SHARED_LIB): $(OBJS)
examples: $(STATIC_LIB) $(EXAMPLES) examples/common.o examples: $(STATIC_LIB) $(EXAMPLES) examples/common.o
soak: examples/soak.c
examples/soak
mls: examples/mls mls: examples/mls
test: $(TESTS) test: $(TESTS)

14
examples/soak.c
View file

@ -538,13 +538,11 @@ verify_sm_is_first_available_span(sparsemap_t *map, sparsemap_idx_t idx, size_t
{ {
for (sparsemap_idx_t i = 0; i < idx + len; i++) { for (sparsemap_idx_t i = 0; i < idx + len; i++) {
sparsemap_idx_t j = 0; sparsemap_idx_t j = 0;
while (sparsemap_is_set(map, i + j) == value && j < len && j < idx + len) { while (sparsemap_is_set(map, i + j) == value && j < len) {
j++; j++;
} }
if (j == len) {
return i == idx; return i == idx;
} }
}
return false; return false;
} }
@ -669,7 +667,6 @@ main(void)
assert(_sparsemap_set(&map, pg, true) == pg); assert(_sparsemap_set(&map, pg, true) == pg);
} }
mdb_midl_sort(list); mdb_midl_sort(list);
stats(0, map, list);
assert(verify_sm_eq_ml(map, list)); assert(verify_sm_eq_ml(map, list));
double b, e; double b, e;
@ -774,8 +771,8 @@ main(void)
assert(verify_sm_eq_ml(map, list)); assert(verify_sm_eq_ml(map, list));
// Once we've used half of the free list, let's replenish it a bit. // Once we've used a tenth of the free list, let's replenish it a bit.
if (list[0] < amt / 2) { if (list[0] < amt / 10) {
do { do {
pgno_t pgno; pgno_t pgno;
size_t len, retries = amt; size_t len, retries = amt;
@ -817,7 +814,7 @@ main(void)
// every so often, either ... // every so often, either ...
if (iterations % 1000 == 0) { if (iterations % 1000 == 0) {
larger_please:; larger_please:;
size_t COUNT = xorshift32() % 1024 + 513; size_t COUNT = xorshift32() % 3586 + 513;
// ... add some amount of 4KiB pages, or // ... add some amount of 4KiB pages, or
size_t len = COUNT; size_t len = COUNT;
// The largest page is at list[1] because this is a reverse sorted list. // The largest page is at list[1] because this is a reverse sorted list.
@ -873,8 +870,9 @@ main(void)
} }
} }
} }
iterations++;
stats(iterations, map, list); stats(iterations, map, list);
// printf("\033[K%zu\r", iterations);
iterations++;
} }
return 0; return 0;

22
src/sparsemap.c
View file

@ -38,7 +38,6 @@
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic" #pragma GCC diagnostic ignored "-Wpedantic"
#pragma GCC diagnostic ignored "-Wvariadic-macros" #pragma GCC diagnostic ignored "-Wvariadic-macros"
#include <stdarg.h>
#define __sm_diag(format, ...) __sm_diag_(__FILE__, __LINE__, __func__, format, ##__VA_ARGS__) #define __sm_diag(format, ...) __sm_diag_(__FILE__, __LINE__, __func__, format, ##__VA_ARGS__)
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
void __attribute__((format(printf, 4, 5))) __sm_diag_(const char *file, int line, const char *func, const char *format, ...) void __attribute__((format(printf, 4, 5))) __sm_diag_(const char *file, int line, const char *func, const char *format, ...)
@ -236,7 +235,7 @@ __sm_chunk_map_set_capacity(__sm_chunk_t *map, size_t capacity)
size_t reduced = 0; size_t reduced = 0;
register uint8_t *p = (uint8_t *)map->m_data; register uint8_t *p = (uint8_t *)map->m_data;
for (ssize_t i = sizeof(sm_bitvec_t) - 1; i >= 0; i--) { // TODO: for (ssize_t i = sizeof(sm_bitvec_t) - 1; i >= 0; i--) {
for (int j = SM_FLAGS_PER_INDEX_BYTE - 1; j >= 0; j--) { for (int j = SM_FLAGS_PER_INDEX_BYTE - 1; j >= 0; j--) {
p[i] &= ~((sm_bitvec_t)SM_PAYLOAD_ONES << (j * 2)); p[i] &= ~((sm_bitvec_t)SM_PAYLOAD_ONES << (j * 2));
p[i] |= ((sm_bitvec_t)SM_PAYLOAD_NONE << (j * 2)); p[i] |= ((sm_bitvec_t)SM_PAYLOAD_NONE << (j * 2));
@ -1105,7 +1104,7 @@ sparsemap_set(sparsemap_t *map, sparsemap_idx_t idx, bool value)
__sm_append_data(map, &buf[0], sizeof(buf)); __sm_append_data(map, &buf[0], sizeof(buf));
uint8_t *p = __sm_get_chunk_map_data(map, 0); uint8_t *p = __sm_get_chunk_map_data(map, 0);
*(sm_idx_t *)p = __sm_get_fully_aligned_offset(idx); // TODO was not fully aligned before... why? *(sm_idx_t *)p = __sm_get_fully_aligned_offset(idx);
__sm_set_chunk_map_count(map, 1); __sm_set_chunk_map_count(map, 1);
@ -1582,18 +1581,18 @@ sparsemap_span(sparsemap_t *map, sparsemap_idx_t idx, size_t len, bool value)
{ {
size_t rank, nth; size_t rank, nth;
sm_bitvec_t vec = 0; sm_bitvec_t vec = 0;
sparsemap_idx_t offset = 0; sparsemap_idx_t offset;
/* When skipping forward to `idx` offset in the map we can determine how /* When skipping forward to `idx` offset in the map we can determine how
many selects we can avoid by taking the rank of the range and starting many selects we can avoid by taking the rank of the range and starting
at that bit. */ at that bit. */
nth = (idx < 1) ? 0 : sparsemap_rank(map, 0, idx - 1, value); nth = (idx == 0) ? 0 : sparsemap_rank(map, 0, idx - 1, value);
/* Find the first bit that matches value, then... */ /* Find the first bit that matches value, then... */
offset = sparsemap_select(map, nth, value); offset = sparsemap_select(map, nth, value);
do { do {
/* See if the rank of the bits in the range starting at offset is equal /* See if the rank of the bits in the range starting at offset is equal
to the desired amount. */ to the desired amount. */
rank = len == 1 ? 1 : sparsemap_rank_vec(map, offset, offset + len - 1, value, &vec); rank = (len == 1) ? 1 : sparsemap_rank_vec(map, offset, offset + len - 1, value, &vec);
if (rank >= len) { if (rank >= len) {
/* We've found what we're looking for, return the index of the first /* We've found what we're looking for, return the index of the first
bit in the range. */ bit in the range. */
@ -1602,13 +1601,10 @@ sparsemap_span(sparsemap_t *map, sparsemap_idx_t idx, size_t len, bool value)
/* Now we try to jump forward as much as possible before we look for a /* Now we try to jump forward as much as possible before we look for a
new match. We do this by counting the remaining bits in the returned new match. We do this by counting the remaining bits in the returned
vec from the call to rank_vec(). */ vec from the call to rank_vec(). */
int amt = 0; int amt = 1;
if (vec == 0) { if (vec > 0) {
/* The returned vec had no set bits, let's move forward in the map. */ /* The returned vec had som set bits, let's move forward in the map as much
amt = (rank == 0) ? len : 1; as possible (max: 64 bit positions). */
} else {
/* We might be able to jump forward up to 64 bit positions saving us repeated
calls to select()/rank(). */
int max = len > SM_BITS_PER_VECTOR ? SM_BITS_PER_VECTOR : len; int max = len > SM_BITS_PER_VECTOR ? SM_BITS_PER_VECTOR : len;
while (amt < max && (vec & 1 << amt)) { while (amt < max && (vec & 1 << amt)) {
amt++; amt++;

2
tests/common.c
View file

@ -87,7 +87,7 @@ xorshift32_seed(void)
} }
void void
shuffle(int *array, size_t n) // TODO working? shuffle(int *array, size_t n)
{ {
for (size_t i = n - 1; i > 0; --i) { for (size_t i = n - 1; i > 0; --i) {
size_t j = xorshift32() % (i + 1); size_t j = xorshift32() % (i + 1);

2
tests/common.h
View file

@ -1,4 +1,6 @@
#include "../include/sparsemap.h"
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wvariadic-macros" #pragma GCC diagnostic ignored "-Wvariadic-macros"
#define __diag(...) \ #define __diag(...) \

24
tests/test.c
View file

@ -46,7 +46,6 @@ populate_map(sparsemap_t *map, int size, int max_value)
int array[size]; int array[size];
setup_test_array(array, size, max_value); setup_test_array(array, size, max_value);
// TODO ensure_sequential_set(array, size, 10);
shuffle(array, size); shuffle(array, size);
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
sparsemap_set(map, array[i], true); sparsemap_set(map, array[i], true);
@ -995,14 +994,6 @@ test_api_span(const MunitParameter params[], void *data)
located_at = sparsemap_span(map, placed_at / 2, 50, true); located_at = sparsemap_span(map, placed_at / 2, 50, true);
assert_true(located_at == placed_at); assert_true(located_at == placed_at);
/* TODO
sparsemap_clear(map);
placed_at = sm_add_span(map, amt, amt - 1);
located_at = sparsemap_span(map, 0, amt - 1, true);
assert_true(located_at == placed_at);
*/
return MUNIT_OK; return MUNIT_OK;
} }
@ -1070,7 +1061,6 @@ test_scale_lots_o_spans(const MunitParameter params[], void *data)
for (size_t i = 0; i < amt;) { for (size_t i = 0; i < amt;) {
int l = i % 31 + 16; int l = i % 31 + 16;
// TODO: sm_add_span(map, amt, l);
sm_add_span(map, 10000, l); sm_add_span(map, 10000, l);
if (errno == ENOSPC) { if (errno == ENOSPC) {
map = sparsemap_set_data_size(map, sparsemap_get_capacity(map) * 2, NULL); map = sparsemap_set_data_size(map, sparsemap_get_capacity(map) * 2, NULL);
@ -1250,8 +1240,6 @@ test_scale_best_case(const MunitParameter params[], void *data)
So, in a 1KiB buffer you have: So, in a 1KiB buffer you have:
(1024 KiB / 8 bytes) * 2048 = 268,435,456 bits (1024 KiB / 8 bytes) * 2048 = 268,435,456 bits
or 1.09 TiB of 4KiB pages. Let's investigate, and find out if that's the case. or 1.09 TiB of 4KiB pages. Let's investigate, and find out if that's the case.
TODO: Actually, 172032 are stored before SEGV, or 706 MiB of 4KiB pages.
*/ */
/* Set every bit on, that should be the best case. */ /* Set every bit on, that should be the best case. */
@ -1296,8 +1284,6 @@ test_scale_worst_case(const MunitParameter params[], void *data)
So, in a 1KiB buffer you have: So, in a 1KiB buffer you have:
(1024 KiB / 264 bytes) * 2048 = 8,134,407.75758 bits (1024 KiB / 264 bytes) * 2048 = 8,134,407.75758 bits
or 33.3 GiB of 4KiB pages. Let's investigate, and find out if that's the case. or 33.3 GiB of 4KiB pages. Let's investigate, and find out if that's the case.
TODO: actually 7744 are stored before SEGV, or 31MiB of 4KiB pages.
*/ */
/* Set every other bit, that has to be the "worst case" for this index. */ /* Set every other bit, that has to be the "worst case" for this index. */
@ -1336,26 +1322,16 @@ static MunitResult
test_perf_span_solo(const MunitParameter params[], void *data) test_perf_span_solo(const MunitParameter params[], void *data)
{ {
sparsemap_t *map = (sparsemap_t *)data; sparsemap_t *map = (sparsemap_t *)data;
// double stop, start;
(void)params; (void)params;
int located_at, placed_at, amt = 500; int located_at, placed_at, amt = 500;
assert_ptr_not_null(map); assert_ptr_not_null(map);
return MUNIT_OK; // TODO
for (int i = 1; i < amt; i++) { for (int i = 1; i < amt; i++) {
for (int length = 1; length <= 100; length++) { for (int length = 1; length <= 100; length++) {
sparsemap_clear(map); sparsemap_clear(map);
placed_at = sm_add_span(map, amt, length); placed_at = sm_add_span(map, amt, length);
// logf("i = %d, length = %d\tplaced_at %d\n", i, length, placed_at);
// sm_whats_set(map, 5000);
// start = nsts();
located_at = sparsemap_span(map, 0, length, true); located_at = sparsemap_span(map, 0, length, true);
// stop = nsts();
// double amt = (stop - start) * 1e6;
// if (amt > 0) {
// fprintf(stdout, "%0.8f\n", amt);
// }
if (placed_at != located_at) if (placed_at != located_at)
logf("a: i = %d, length = %d\tplaced_at %d located_at %d\n", i, length, placed_at, located_at); logf("a: i = %d, length = %d\tplaced_at %d located_at %d\n", i, length, placed_at, located_at);
} }