.idea/customTargets.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CLionExternalBuildManager">
+    <target id="db0ccaeb-4851-470b-83d0-afa663f6ceb9" name="tests/soak" defaultType="MAKE">
+      <configuration id="98973a90-a9d0-431b-9071-9ce6960b0b01" name="tests/soak">
+        <build type="MAKE">
+          <make targetName="tests/soak" />
+        </build>
+        <clean type="MAKE">
+          <make targetName="clean" />
+        </clean>
+      </configuration>
+    </target>
+  </component>
+</project>

.idea/makefile.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MakefileBuildTargetsManager">
+    <user-build-targets>
+      <build-target name="tests/soak">
+        <build-configurations>
+          <build-configuration>
+            <make-targets>
+              <make-target>tests/soak</make-target>
+            </make-targets>
+          </build-configuration>
+        </build-configurations>
+      </build-target>
+      <build-target name="clean">
+        <build-configurations>
+          <build-configuration>
+            <make-targets>
+              <make-target>clean</make-target>
+            </make-targets>
+          </build-configuration>
+        </build-configurations>
+      </build-target>
+    </user-build-targets>
+  </component>
+</project>

Makefile

@@ -5,8 +5,8 @@ SHARED_LIB = libsparsemap.so
 
 LIBS =  -lm
 #CFLAGS = -Wall -Wextra -Wpedantic -Of -std=c11 -Iinclude/ -fPIC
-CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
-#CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC
+#CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
+CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -fPIC
 #CFLAGS = -DSPARSEMAP_DIAGNOSTIC -DDEBUG -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC
 #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
 #CFLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -fPIC
@@ -14,8 +14,8 @@ CFLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -fPIC
 #CFLAGS = -Wall -Wextra -Wpedantic -Og -g -fsanitize=all -fhardened -std=c11 -Iinclude/ -fPIC
 
 #TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -O0 -g -std=c11 -Iinclude/ -Itests/ -fPIC
-#TEST_FLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -Itests/ -fPIC
-TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC
+#TEST_FLAGS = -Wall -Wextra -Wpedantic -Ofast -g -std=c11 -Iinclude/ -Itests/ -fPIC
+TEST_FLAGS = -Wall -Wextra -Wpedantic -Og -g -std=c11 -Iinclude/ -Itests/ -fPIC
 #TEST_FLAGS = -DDEBUG -Wall -Wextra -Wpedantic -Og -g -fsanitize=address,leak,object-size,pointer-compare,pointer-subtract,null,return,bounds,pointer-overflow,undefined -fsanitize-address-use-after-scope -std=c11 -Iinclude/ -fPIC
 
 TESTS = tests/test tests/soak
@@ -50,7 +50,7 @@ soak: tests
 	env ASAN_OPTIONS=detect_leaks=1 LSAN_OPTIONS=verbosity=1:log_threads=1 ./tests/soak
 
 tests/test: $(TEST_OBJS) $(LIB_OBJS) $(STATIC_LIB)
-	$(CC) $^ -o $@ $(TEST_FLAGS)
+	$(CC) $^ $(LIBS) -o $@ $(TEST_FLAGS)
 
 clean:
 	rm -f $(OBJS)

include/sparsemap.h

@@ -297,14 +297,16 @@ sparsemap_idx_t sparsemap_get_ending_offset(sparsemap_t *map);
  */
 void sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t vec[], size_t n, void *aux), size_t skip, void *aux);
 
-/** @brief Merges the values from \b other into \b map, \b other is unchanged.
+/** @brief Merges the values from \b source into \b destination, \b source is unchanged.
  *
- * @param[in] map The sparsemap reference.
- * @param[in] other The bitmap to merge into \b map.
+ * Efficiently adds all set bits from \b source into \b destination.
+ *
+ * @param[in] destination The sparsemap reference into which we will merge \b source.
+ * @param[in] source The bitmap to merge into \b destination.
  * @returns 0 on success, or sets errno to ENOSPC and returns the amount of
  * additional space required to successfully merge the maps.
  */
-int sparsemap_merge(sparsemap_t *map, sparsemap_t *other);
+int sparsemap_merge(sparsemap_t *destination, sparsemap_t *source);
 
 /** @brief Splits the bitmap by assigning all bits starting at \b offset to the
  * \b other bitmap while removing them from \b map.

src/sparsemap.c

@@ -228,9 +228,9 @@ static void
 __sm_chunk_set_capacity(__sm_chunk_t *chunk, size_t capacity)
 {
   __sm_assert(capacity % SM_BITS_PER_VECTOR == 0);
-  __sm_assert(capacity < SM_CHUNK_MAX_CAPACITY);
+  __sm_assert(capacity <= SM_CHUNK_MAX_CAPACITY);
 
-  if (capacity >= SM_CHUNK_MAX_CAPACITY) {
+  if (capacity > SM_CHUNK_MAX_CAPACITY) {
     return;
   }
 
@@ -426,15 +426,23 @@ __sm_chunk_set(__sm_chunk_t *chunk, size_t idx, bool value, size_t *pos, sm_bitv
 /** @brief Merges into the chunk at \b offset all set bits from \b src.
  *
  * @param[in] chunk The chunk in question.
- * @param[in] offset The fully aligned offset of the chunk to be merged.
+ * @param[in] offset The offset of the first bit in the chunk to be merged.
+ * @todo merge vectors rather than call sparsemap_set() in a loop
  */
 void
-__sm_merge_chunk(sparsemap_t *chunk, sparsemap_idx_t offset, __sm_chunk_t src)
+__sm_merge_chunk(sparsemap_t *map, sparsemap_idx_t offset, __sm_chunk_t dst_chunk, __sm_chunk_t src_chunk)
 {
-  size_t capacity = __sm_chunk_get_capacity(&src);
-  for (sparsemap_idx_t j = 0; j < capacity; j++, offset++) {
-    if (__sm_chunk_is_set(&src, j)) {
-      sparsemap_set(chunk, offset, true);
+  size_t src_capacity = __sm_chunk_get_capacity(&src_chunk);
+  (void)dst_chunk;
+#if 0
+  size_t dst_capacity = __sm_chunk_get_capacity(&dst_chunk);
+  if (dst_capacity < src_capacity) {
+    __sm_chunk_set_capacity(&dst_chunk, src_capacity);
+  }
+#endif
+  for (sparsemap_idx_t j = 0; j < src_capacity; j++) {
+    if (__sm_chunk_is_set(&src_chunk, j)) {
+      sparsemap_set(map, offset + j, true);
     }
   }
 }
@@ -1125,7 +1133,7 @@ sparsemap_set(sparsemap_t *map, sparsemap_idx_t idx, bool value)
     __sm_append_data(map, &buf[0], sizeof(buf));
 
     uint8_t *p = __sm_get_chunk_data(map, 0);
-    *(sm_idx_t *)p = __sm_get_chunk_aligned_offset(idx); // TODO: this was get_vector_aligned_offset
+    *(sm_idx_t *)p = __sm_get_vector_aligned_offset(idx);
 
     __sm_set_chunk_count(map, 1);
 
@@ -1396,22 +1404,22 @@ sparsemap_scan(sparsemap_t *map, void (*scanner)(sm_idx_t[], size_t, void *aux),
 }
 
 int
-sparsemap_merge(sparsemap_t *map, sparsemap_t *other)
+sparsemap_merge(sparsemap_t *destination, sparsemap_t *source)
 {
   uint8_t *src, *dst;
-  size_t src_count = __sm_get_chunk_count(other);
-  size_t dst_count = __sm_get_chunk_count(map);
+  size_t src_count = __sm_get_chunk_count(source);
+  size_t dst_count = __sm_get_chunk_count(destination);
   size_t max_chunk_count = src_count + dst_count;
-  ssize_t difference = map->m_capacity - (other->m_data_used + src_count * (sizeof(sm_idx_t) + sizeof(sm_bitvec_t) * 2));
+  ssize_t remaining_capacity = destination->m_capacity - (source->m_data_used + src_count * (sizeof(sm_idx_t) + sizeof(sm_bitvec_t) * 2));
 
   /* Estimate worst-case overhead required for merge. */
-  if (difference <= 0) {
+  if (remaining_capacity <= 0) {
     errno = ENOSPC;
-    return -difference;
+    return -remaining_capacity;
   }
 
-  dst = __sm_get_chunk_data(map, 0);
-  src = __sm_get_chunk_data(other, 0);
+  src = __sm_get_chunk_data(source, 0);
+  dst = __sm_get_chunk_data(destination, 0);
   for (size_t i = 0; i < max_chunk_count && src_count; i++) {
     sm_idx_t src_start = *(sm_idx_t *)src;
     sm_idx_t dst_start = *(sm_idx_t *)dst;
@@ -1430,8 +1438,7 @@ sparsemap_merge(sparsemap_t *map, sparsemap_t *other)
       __sm_chunk_init(&src_chunk, src + sizeof(sm_idx_t));
       __sm_chunk_t dst_chunk;
       __sm_chunk_init(&dst_chunk, dst + sizeof(sm_idx_t));
-      __sm_merge_chunk(map, src_start, src_chunk);
-      *(sm_idx_t *)dst = __sm_get_chunk_aligned_offset(src_start);
+      __sm_merge_chunk(destination, *(sm_idx_t *)src, dst_chunk, src_chunk);
       src += sizeof(sm_idx_t) + __sm_chunk_get_size(&src_chunk);
       dst += sizeof(sm_idx_t) + __sm_chunk_get_size(&dst_chunk);
       dst_count--;
@@ -1443,14 +1450,14 @@ sparsemap_merge(sparsemap_t *map, sparsemap_t *other)
       __sm_chunk_init(&src_chunk, src + sizeof(sm_idx_t));
       size_t src_size = __sm_chunk_get_size(&src_chunk);
       if (dst_count == 0) {
-        __sm_append_data(map, src, sizeof(sm_idx_t) + src_size);
+        __sm_append_data(destination, src, sizeof(sm_idx_t) + src_size);
       } else {
-        size_t offset = __sm_get_chunk_offset(map, dst_start);
-        __sm_insert_data(map, offset, src, sizeof(sm_idx_t) + src_size);
+        size_t offset = __sm_get_chunk_offset(destination, dst_start);
+        __sm_insert_data(destination, offset, src, sizeof(sm_idx_t) + src_size);
       }
 
       /* Update the chunk count and data_used. */
-      __sm_set_chunk_count(map, __sm_get_chunk_count(map) + 1);
+      __sm_set_chunk_count(destination, __sm_get_chunk_count(destination) + 1);
 
       /* Carry on to the next chunk. */
       __sm_chunk_t dst_chunk;
@@ -1528,7 +1535,7 @@ sparsemap_split(sparsemap_t *map, sparsemap_idx_t offset, sparsemap_t *other)
     uint8_t buf[sizeof(sm_idx_t) + sizeof(sm_bitvec_t) * 2] = { 0 };
     memcpy(dst, &buf[0], sizeof(buf));
 
-    *(sm_idx_t *)dst = __sm_get_chunk_aligned_offset(offset); //TODO: was simply "offset"
+    *(sm_idx_t *)dst = __sm_get_vector_aligned_offset(offset); // TODO: was simply "offset"
     dst += sizeof(sm_idx_t);
 
     /* the |other| sparsemap_t now has one additional chunk */
@@ -1551,7 +1558,7 @@ sparsemap_split(sparsemap_t *map, sparsemap_idx_t offset, sparsemap_t *other)
     for (size_t j = offset % capacity; j < capacity; j++, d++) {
       if (__sm_chunk_is_set(&s_chunk, j)) {
         sparsemap_set(other, d, true);
-        sparsemap_set(map, d, false); //TODO remove, and fix set_capacity below
+        sparsemap_set(map, d, false); // TODO remove, and fix set_capacity below
       }
     }
 

tests/soak.c

@@ -6,9 +6,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "../include/sparsemap.h"
 #include "../include/common.h"
 #include "../include/roaring.h"
-#include "../include/sparsemap.h"
 #include "../include/tdigest.h"
 
 /* midl.h ------------------------------------------------------------------ */

tests/test.c

@@ -749,7 +749,7 @@ test_api_split(const MunitParameter params[], void *data)
   }
 
   sparsemap_idx_t offset = sparsemap_split(map, SPARSEMAP_IDX_MAX, &portion);
-  for (int i = 0; i < offset; i++) {
+  for (size_t i = 0; i < offset; i++) {
     assert_true(sparsemap_is_set(map, i));
     assert_false(sparsemap_is_set(&portion, i));
   }
@@ -767,7 +767,7 @@ test_api_split(const MunitParameter params[], void *data)
   }
 
   offset = sparsemap_split(map, SPARSEMAP_IDX_MAX, &portion);
-  for (int i = 0; i < offset - 24; i++) {
+  for (size_t i = 0; i < offset - 24; i++) {
     assert_true(sparsemap_is_set(map, i + 24));
     assert_false(sparsemap_is_set(&portion, i + 24));
   }
@@ -842,11 +842,11 @@ test_api_merge(const MunitParameter params[], void *data)
 
   sparsemap_clear(map);
   sparsemap_clear(other);
-
   sparsemap_set(other, 1, true);
   sparsemap_set(other, 2049, true);
   sparsemap_set(map, 2050, true);
   sparsemap_set(other, 4097, true);
+  sparsemap_set(map, 6113, true);
   sparsemap_set(other, 8193, true);
 
   sparsemap_merge(map, other);
@@ -855,10 +855,11 @@ test_api_merge(const MunitParameter params[], void *data)
   assert_true(sparsemap_is_set(map, 2049));
   assert_true(sparsemap_is_set(map, 2050));
   assert_true(sparsemap_is_set(map, 4097));
+  assert_true(sparsemap_is_set(map, 6113));
   assert_true(sparsemap_is_set(map, 8193));
 
   for (int i = 0; i < 10000; i++) {
-    if (i == 2049 || i == 1 || i == 2050 || i == 4097 || i == 8193)
+    if (i == 2049 || i == 1 || i == 2050 || i == 4097 || i == 6113 || i == 8193)
       continue;
     else
       assert_false(sparsemap_is_set(map, i));