fixes

examples/ex_2.c

@@ -38,7 +38,7 @@ main(void)
     }
   }
   // On 1024 KiB of buffer with every other bit set the map holds 7744 bits
-  // and then runs out of space.  This next _set() call will fail/abort.
+  // and then runs out of space.  This next _set() call will fail.
   sparsemap_set(map, ++i, true);
   assert(sparsemap_is_set(map, i) == true);
   return 0;

examples/soak.c

@@ -75,6 +75,13 @@ void mdb_midl_free(MDB_IDL ids);
  */
 void mdb_midl_shrink(MDB_IDL *idp);
 
+/** Shrink an IDL to a specific size.
+ * Resize the IDL to \b size if it is larger.
+ * @param[in,out] idp	Address of the IDL to shrink.
+ * @param[in] size	Capacity to have once resized.
+ */
+void mdb_midl_shrink(MDB_IDL *idp);
+
 /** Make room for num additional elements in an IDL.
  * @param[in,out] idp	Address of the IDL.
  * @param[in] num	Number of elements to make room for.
@@ -218,6 +225,17 @@ mdb_midl_shrink(MDB_IDL *idp)
   }
 }
 
+void
+mdb_midl_shrink_to(MDB_IDL *idp, size_t size)
+{
+  MDB_IDL ids = *idp;
+  if (*(--ids) > size && (ids = realloc(ids, (size + 2) * sizeof(MDB_ID)))) {
+    *ids++ = size;
+    *idp = ids;
+    *idp[0] = *idp[0] > size ? size : *idp[0];
+  }
+}
+
 static int
 mdb_midl_grow(MDB_IDL *idp, int num)
 {
@@ -426,8 +444,8 @@ toss(size_t max)
 bool
 verify_midl_contains(MDB_IDL list, pgno_t pg)
 {
-  unsigned index = mdb_midl_search(list, pg);
-  return index <= list[0] && list[index] == pg;
+  unsigned idx = mdb_midl_search(list, pg);
+  return idx <= list[0] && list[idx] == pg;
 }
 
 bool
@@ -445,8 +463,8 @@ verify_midl_nodups(MDB_IDL list)
 bool
 verify_span_midl(MDB_IDL list, pgno_t pg, unsigned len)
 {
-  pgno_t f = mdb_midl_search(list, pg);
-  bool found = (list[f] == pg) && (f <= list[0]);
+  pgno_t idx = mdb_midl_search(list, pg);
+  bool found = idx <= list[0] && list[idx] == pg;
   if (!found)
     return false;
   if (len == 1)
@@ -460,8 +478,8 @@ bool
 verify_empty_midl(MDB_IDL list, pgno_t pg, unsigned len)
 {
   for (pgno_t i = pg; i < pg + len; i++) {
-    pgno_t f = mdb_midl_search(list, pg);
-    bool found = list[f] == pg && f <= list[0];
+    pgno_t idx = mdb_midl_search(list, pg);
+    bool found = idx <= list[0] && list[idx] == pg;
     if (found)
       return false;
   }
@@ -493,22 +511,41 @@ verify_empty_sparsemap(sparsemap_t *map, pgno_t pg, unsigned len)
 bool
 verify_sm_eq_ml(sparsemap_t *map, MDB_IDL list)
 {
-  for (unsigned i = 0; i <= list[1]; i++) {
-    pgno_t f = mdb_midl_search(list, i);
-    bool found = list[f] == i && f <= list[0];
-    if (sparsemap_is_set(map, i) != found)
+  for (int i = 1; i <= list[0]; i++) {
+    pgno_t pg = list[i];
+    unsigned skipped = i == 1 ? 0 : list[i-1] - list[i] - 1;
+    for (int j = 0; j < skipped; j++) {
+      if (sparsemap_is_set(map, pg - j) != false)
+        return false;
+    }
+    if (sparsemap_is_set(map, pg) != true)
       return false;
   }
   return true;
 }
 
 void
-print_sizes(sparsemap_t *map, MDB_IDL list)
+stats(size_t iterations, sparsemap_t *map, MDB_IDL list)
 {
   char m[1024], l[1024];
-  __diag("idl: %s bytes\tsm: %s bytes\n", bytes_as(MDB_IDL_SIZEOF(list), m, 1024), bytes_as(sparsemap_get_capacity(map), l, 1024));
+  __diag("%zu\tidl[%zu/%zu]: %s\tsm: %s\n", iterations, list[-1], list[0], bytes_as(MDB_IDL_SIZEOF(list), m, 1024), bytes_as(sparsemap_get_capacity(map), l, 1024));
 }
 
+sparsemap_idx_t
+_sparsemap_set(sparsemap_t **map, sparsemap_idx_t idx, bool value)
+{
+  sparsemap_idx_t l = sparsemap_set(*map, idx, value);
+  if (errno == ENOSPC) {
+    *map = sparsemap_set_data_size(*map, sparsemap_get_capacity(*map) + 64, NULL);
+    assert(*map != NULL);
+    errno = 0;
+  }
+  return l;
+}
+
+
+#define INITIAL_AMOUNT 1024 * 2
+
 /*
  * A "soak test" that tries to replicate behavior in LMDB for page allocation.
  */
@@ -524,7 +561,7 @@ main()
 
   __diag("starting...\n");
 
-  size_t amt = 1024 * 2; // 1024 * 1024 * 2;
+  size_t amt = INITIAL_AMOUNT;
   MDB_IDL list = mdb_midl_alloc(amt);
   sparsemap_t *map = sparsemap(3 * 1024);
 
@@ -533,18 +570,20 @@ main()
   //  - Sparsemap will compress the set bits using less memory
   mdb_midl_need(&list, amt);
   for (size_t pg = 0; pg < amt; pg++) {
-    mdb_midl_xappend(list, pg);   // listed page ids are free
-    sparsemap_set(map, pg, true); // true means free in our bitmap
+    // We list every free (unallocated) page in the IDL, while...
+    mdb_midl_xappend(list, pg);
+    // ... true (unset in the bitmap) indicates free in the bitmap.
+    assert(_sparsemap_set(&map, pg, true) == pg);
   }
   mdb_midl_sort(list);
-  print_sizes(map, list);
+  stats(0, map, list);
   assert(verify_sm_eq_ml(map, list));
 
   while (1) {
     unsigned mi;
     pgno_t ml = 0, sl = 0;
 
-    // get an amount [1, 16] of pages to find prefering smaller sizes
+    // get an amount [1, 16] of pages to find preferring smaller sizes
     unsigned n = toss(15) + 1;
 
     // find a set of pages using the MDB_IDL
@@ -610,11 +649,11 @@ main()
     // acquire the set of pages within the sparsemap
     if (prefer_mdb_idl_location) {
       for (pgno_t i = ml; i < ml + n; i++) {
-        sparsemap_set(map, i, false);
+        assert(_sparsemap_set(&map, i, false) == i);
       }
     } else {
       for (pgno_t i = sl; i <= sl + n; i++) {
-        sparsemap_set(map, i, false);
+        assert(_sparsemap_set(&map, i, false) == i);
       }
     }
 
@@ -632,12 +671,12 @@ main()
         if (SPARSEMAP_FOUND(pg)) {
           assert(verify_empty_midl(list, pg, len));
           assert(verify_empty_sparsemap(map, pg, len));
+          if (list[-1] - list[0] < len)
+            mdb_midl_need(&list, list[-1] + len);
           for (int i = pg; i < pg + len; i++) {
-            if (pg + len > list[-1])
-              mdb_midl_need(&list, pg + len);
             assert(verify_midl_contains(list, i) == false);
             mdb_midl_insert(list, i);
-            sparsemap_set(map, i, true);
+            assert(_sparsemap_set(&map, i, true) == i);
           }
           mdb_midl_sort(list);
           assert(verify_midl_nodups(list));
@@ -646,7 +685,41 @@ main()
         }
       } while (list[0] < amt - 32);
     }
-    print_sizes(map, list);
+    stats(iterations, map, list);
+
+    // every 100 iterations, either ...
+    if (iterations % 100 == 0) {
+      const int COUNT = 1024;
+      if (toss(6) + 1 < 7) {
+        // ... add a MiB of 4KiB pages, or
+        int len = COUNT;
+        // The largest page is at list[1] because this is a reverse sorted list.
+        int pg = list[1] + 1;
+        if (list[0] + COUNT > list[-1])
+          mdb_midl_grow(&list, list[0] + len);
+        for (int i = pg; i < pg + len; i++) {
+          assert(verify_midl_contains(list, i) == false);
+          assert(sparsemap_is_set(map, i) == false);
+          mdb_midl_insert(list, i);
+          assert(_sparsemap_set(&map, i, true) == i);
+        }
+        mdb_midl_sort(list);
+        assert(verify_midl_nodups(list));
+        verify_sm_eq_ml(map, list);
+      } else {
+        if (list[-1] > INITIAL_AMOUNT) {
+          // ... a fraction of the time, remove a MiB of 4KiB pages.
+          for (int i = 0; i < COUNT; i++) {
+            pgno_t pg = list[list[0] - i];
+            assert(sparsemap_is_set(map, pg) == true);
+            assert(_sparsemap_set(&map, pg, false) == pg) ;
+          }
+          mdb_midl_shrink_to(&list, list[0] - COUNT);
+          assert(verify_midl_nodups(list));
+          verify_sm_eq_ml(map, list);
+        }
+      }
+    }
     iterations++;
   }
 

src/sparsemap.c

@@ -1338,7 +1338,7 @@ sparsemap_select(sparsemap_t *map, sparsemap_idx_t n, bool value)
     if (value) {
       return SPARSEMAP_IDX_MAX;
     } else {
-      return count * SM_CHUNK_MAX_CAPACITY + 1;
+      return count * SM_CHUNK_MAX_CAPACITY;
     }
   } else {
     // TODO... sparsemap_select(map, -n, value); seek from end, not start