Makefile

@@ -60,7 +60,7 @@ clean:
 	rm -f $(EXAMPLES) examples/*.o
 
 format:
-	clang-format -i src/sparsemap.c include/sparsemap.h examples/ex_*.c tests/soak.c tests/test.c lib/common.c include/common.h
+	clang-format -i src/sparsemap.c include/sparsemap.h examples/ex_*.c tests/soak.c tests/test.c tests/midl.c lib/common.c include/common.h
 #	clang-format -i include/*.h src/*.c tests/*.c tests/*.h examples/*.c
 
 %.o: src/%.c

include/bencode.h

@@ -0,0 +1,141 @@
+// https://github.com/willemt/heapless-bencode
+//
+#ifndef BENCODE_H_
+#define BENCODE_H_
+
+typedef struct
+{
+    const char *str;
+    const char *start;
+    void *parent;
+    int val;
+    int len;
+} bencode_t;
+
+/**
+* Initialise a bencode object.
+* @param be The bencode object
+* @param str Buffer we expect input from
+* @param len Length of buffer
+*/
+void bencode_init(
+    bencode_t * be,
+    const char *str,
+    int len
+);
+
+/**
+* @return 1 if the bencode object is an int; otherwise 0.
+*/
+int bencode_is_int(
+    const bencode_t * be
+);
+
+/**
+* @return 1 if the bencode object is a string; otherwise 0.
+*/
+int bencode_is_string(
+    const bencode_t * be
+);
+
+/**
+* @return 1 if the bencode object is a list; otherwise 0.
+*/
+int bencode_is_list(
+    const bencode_t * be
+);
+
+/**
+* @return 1 if the bencode object is a dict; otherwise 0.
+*/
+int bencode_is_dict(
+    const bencode_t * be
+);
+
+/**
+* Obtain value from integer bencode object.
+* @param val Long int we are writing the result to
+* @return 1 on success, otherwise 0
+*/
+int bencode_int_value(
+    bencode_t * be,
+    long int *val
+);
+
+/**
+* @return 1 if there is another item on this dict; otherwise 0.
+*/
+int bencode_dict_has_next(
+    bencode_t * be
+);
+
+/**
+* Get the next item within this dictionary.
+* @param be_item Next item.
+* @param key Const pointer to key string of next item.
+* @param klen Length of the key of next item.
+* @return 1 on success; otherwise 0.
+*/
+int bencode_dict_get_next(
+    bencode_t * be,
+    bencode_t * be_item,
+    const char **key,
+    int *klen
+);
+
+/**
+* Get the string value from this bencode object.
+* The buffer returned is stored on the stack.
+* @param be The bencode object.
+* @param str Const pointer to the buffer.
+* @param slen Length of the buffer we are outputting.
+* @return 1 on success; otherwise 0
+*/
+int bencode_string_value(
+    bencode_t * be,
+    const char **str,
+    int *len
+);
+
+/**
+* Tell if there is another item within this list.
+* @param be The bencode object
+* @return 1 if another item exists on the list; 0 otherwise; -1 on invalid processing
+*/
+int bencode_list_has_next(
+    bencode_t * be
+);
+
+/**
+* Get the next item within this list.
+* @param be The bencode object
+* @param be_item The next bencode object that we are going to initiate.
+* @return return 0 on end; 1 on have next; -1 on error
+*/
+int bencode_list_get_next(
+    bencode_t * be,
+    bencode_t * be_item
+);
+
+/**
+ * Copy bencode object into other bencode object
+ */
+void bencode_clone(
+    bencode_t * be,
+    bencode_t * output
+);
+
+/**
+* Get the start and end position of this dictionary
+* @param be Bencode object
+* @param start Starting string
+* @param len Length of the dictionary 
+* @return 0 on success
+*/
+int bencode_dict_get_start_and_len(
+    bencode_t * be,
+    const char **start,
+    int *len
+);
+
+#endif /* BENCODE_H_ */

lib/bencode.c

@@ -0,0 +1,511 @@
+
+/**
+ * Copyright (c) 2014, Willem-Hendrik Thiart
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file. 
+ *
+ * @file
+ * @brief Read bencoded data
+ * @author  Willem Thiart himself@willemthiart.com
+ * @version 0.1
+ * https://github.com/willemt/heapless-bencode
+ */
+
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "bencode.h"
+
+/**
+ * Carry length over to a new bencode object.
+ * This is done so that we don't exhaust the buffer */
+static int __carry_length(
+    bencode_t * be,
+    const char *pos
+)
+{
+    assert(0 < be->len);
+    return be->len - (pos - be->str);
+}
+
+/**
+ * @param end The point that we read out to
+ * @param val Output of number represented by string 
+ * @return 0 if error; otherwise 1 */
+static long int __read_string_int(
+    const char *sp,
+    const char **end,
+    long int *val
+)
+{
+    *val = 0;
+
+    int sign = 1;
+
+    /* negative */
+    if ('-' == *sp)
+    {
+        sign = -1;
+        sp++;
+    }
+
+    if (!isdigit(*sp))
+        return 0;
+
+    /* work out number */
+    do
+    {
+        *val *= 10;
+        *val += *sp - '0';
+        sp++;
+    }
+    while (isdigit(*sp));
+
+    *val *= sign;
+
+    *end = sp;
+    return 1;
+}
+
+int bencode_is_dict(
+    const bencode_t * be
+)
+{
+    return be->str && *be->str == 'd';
+}
+
+int bencode_is_int(
+    const bencode_t * be
+)
+{
+    return be->str && *be->str == 'i';
+}
+
+int bencode_is_list(
+    const bencode_t * be
+)
+{
+    return be->str && *be->str == 'l';
+}
+
+int bencode_is_string(
+    const bencode_t * be
+)
+{
+    const char *sp;
+
+    sp = be->str;
+
+    assert(sp);
+
+    if (!isdigit(*sp))
+        return 0;
+
+    do sp++;
+    while (isdigit(*sp));
+
+    return *sp == ':';
+}
+
+/**
+ * Move to next item
+ * @param sp The bencode string we are processing
+ * @return Pointer to string on success, otherwise NULL */
+static const char *__iterate_to_next_string_pos(
+    bencode_t * be,
+    const char *sp
+)
+{
+    bencode_t iter;
+
+    bencode_init(&iter, sp, __carry_length(be, sp));
+
+    if (bencode_is_dict(&iter))
+    {
+        /* navigate to the end of the dictionary */
+        while (bencode_dict_has_next(&iter))
+        {
+            /* ERROR: input string is invalid */
+            if (0 == bencode_dict_get_next(&iter, NULL, NULL, NULL))
+                return NULL;
+        }
+
+        /* special case for empty dict */
+        if (*iter.str == 'd' && *(iter.str + 1) == 'e')
+            return iter.str + 2;
+
+        return iter.str + 1;
+    }
+    else if (bencode_is_list(&iter))
+    {
+        /* navigate to the end of the list */
+        while (bencode_list_has_next(&iter))
+        {
+            /* ERROR: input string is invalid */
+            if (-1 == bencode_list_get_next(&iter, NULL))
+                return NULL;
+        }
+
+        return iter.str + 1;
+    }
+    else if (bencode_is_string(&iter))
+    {
+        int len;
+        const char *str;
+
+        /* ERROR: input string is invalid */
+        if (0 == bencode_string_value(&iter, &str, &len))
+            return NULL;
+
+        return str + len;
+    }
+    else if (bencode_is_int(&iter))
+    {
+        const char *end;
+        long int val;
+
+        if (0 == __read_string_int(&iter.str[1], &end, &val))
+            return NULL;
+
+        assert(end[0] == 'e');
+
+        return end + 1;
+    }
+
+    /* input string is invalid */
+    return NULL;
+}
+
+static const char *__read_string_len(
+    const char *sp,
+    int *slen
+)
+{
+    *slen = 0;
+
+    if (!isdigit(*sp))
+        return NULL;
+
+    do
+    {
+        *slen *= 10;
+        *slen += *sp - '0';
+        sp++;
+    }
+    while (isdigit(*sp));
+
+    assert(*sp == ':');
+    assert(0 <= *slen);
+
+    return sp + 1;
+}
+
+void bencode_init(
+    bencode_t * be,
+    const char *str,
+    const int len
+)
+{
+    memset(be, 0, sizeof(bencode_t));
+    be->str = be->start = str;
+    be->str = str;
+    be->len = len;
+    /* assert(0 < be->len); */
+}
+
+int bencode_int_value(
+    bencode_t * be,
+    long int *val
+)
+{
+    const char *end;
+
+    if (0 == __read_string_int(&be->str[1], &end, val))
+        return 0;
+
+    assert(end[0] == 'e');
+
+    return 1;
+}
+
+int bencode_dict_has_next(
+    bencode_t * be
+)
+{
+    const char *sp = be->str;
+
+    assert(be);
+
+    if (!sp
+        /* at end of dict */
+        || *sp == 'e'
+        /* at end of string */
+        || *sp == '\0'
+        || *sp == '\r'
+        /* empty dict */
+        || (*sp == 'd' && *(sp + 1) == 'e')
+        /* at the end of the input string */
+        || be->str >= be->start + be->len - 1)
+    {
+        return 0;
+    }
+
+    return 1;
+}
+
+int bencode_dict_get_next(
+    bencode_t * be,
+    bencode_t * be_item,
+    const char **key,
+    int *klen
+)
+{
+    const char *sp = be->str;
+    const char *keyin;
+    int len;
+
+    assert(*sp != 'e');
+
+    /* if at start increment to 1st key */
+    if (*sp == 'd')
+    {
+        sp++;
+    }
+
+    /* can't get the next item if we are at the end of the dict */
+    if (*sp == 'e')
+    {
+        return 0;
+    }
+
+    /* 1. find out what the key's length is */
+    keyin = __read_string_len(sp, &len);
+
+    /* 2. if we have a value bencode, lets put the value inside */
+    if (be_item)
+    {
+        *klen = len;
+        bencode_init(be_item, keyin + len, __carry_length(be, keyin + len));
+    }
+
+    /* 3. iterate to next dict key, or move to next item in parent */
+    if (!(be->str = __iterate_to_next_string_pos(be, keyin + len)))
+    {
+        /*  if there isn't anything else or we are at the end of the string */
+        return 0;
+    }
+
+#if 0
+    /*  if at the end of bencode, check that the 'e' terminator is there */
+    if (be->str == be->start + be->len - 1 && *be->str != 'e')
+    {
+        be->str = NULL;
+        return 0;
+    }
+#endif
+
+    assert(be->str);
+
+    if (key)
+    {
+        *key = keyin;
+    }
+
+    return 1;
+}
+
+int bencode_string_value(
+    bencode_t * be,
+    const char **str,
+    int *slen
+)
+{
+    const char *sp;
+
+    *slen = 0;
+    
+    assert(bencode_is_string(be));
+    
+    sp = __read_string_len(be->str, slen);
+    
+    assert(sp);
+    assert(0 < be->len);
+    
+    /*  make sure we still fit within the buffer */
+    if (sp + *slen > be->start + (long int) be->len)
+    {
+        *str = NULL;
+        return 0;
+    }
+
+    *str = sp;
+    return 1;
+}
+
+int bencode_list_has_next(
+    bencode_t * be
+)
+{
+    const char *sp;
+
+    sp = be->str;
+    
+    /* empty list */
+    if (*sp == 'l' &&
+        sp == be->start &&
+        *(sp + 1) == 'e')
+    {
+        be->str++;
+        return 0;
+    }
+
+    /* end of list */
+    if (*sp == 'e')
+    {
+        return 0;
+    }
+
+    return 1;
+}
+
+int bencode_list_get_next(
+    bencode_t * be,
+    bencode_t * be_item
+)
+{
+    const char *sp;
+
+    sp = be->str;
+
+#if 0 /* debugging */
+    printf("%.*s\n", (int)(be->len - (be->str - be->start)), be->str);
+#endif
+
+    /* we're at the end */
+    if (!sp || *sp == 'e')
+        return 0;
+
+    if (*sp == 'l')
+    {
+        /* just move off the start of this list */
+        if (be->start == be->str)
+        {
+            sp++;
+        }
+    }
+
+    /* can't get the next item if we are at the end of the list */
+    if (*sp == 'e')
+    {
+        be->str = sp;
+        return 0;
+    }
+
+    /* populate the be_item if it is available */
+    if (be_item)
+    {
+        bencode_init(be_item, sp, __carry_length(be, sp));
+    }
+
+    /* iterate to next value */
+    if (!(be->str = __iterate_to_next_string_pos(be, sp)))
+    {
+        return -1;
+    }
+
+    return 1;
+}
+
+void bencode_clone(
+    bencode_t * be,
+    bencode_t * output
+)
+{
+    memcpy(output, be, sizeof(bencode_t));
+}
+
+int bencode_dict_get_start_and_len(
+    bencode_t * be,
+    const char **start,
+    int *len
+)
+{
+    bencode_t ben, ben2;
+    const char *ren;
+    int tmplen;
+
+    bencode_clone(be, &ben);
+    *start = ben.str;
+    while (bencode_dict_has_next(&ben))
+        bencode_dict_get_next(&ben, &ben2, &ren, &tmplen);
+
+    *len = ben.str - *start + 1;
+    return 0;
+}
+
+static int __validate(bencode_t *ben)
+{
+    if (bencode_is_dict(ben))
+    {
+        while (bencode_dict_has_next(ben))
+        {
+            int klen;
+            const char *key;
+            bencode_t benk;
+
+            if (0 == bencode_dict_get_next(ben, &benk, &key, &klen))
+                return -1;
+
+            int ret = __validate(&benk);
+            if (0 != ret)
+                return ret;
+        }
+    }
+    else if (bencode_is_list(ben))
+    {
+        while (bencode_list_has_next(ben))
+        {
+            bencode_t benl;
+
+            if (-1 == bencode_list_get_next(ben, &benl))
+                return -1;
+
+            int ret = __validate(&benl);
+            if (0 != ret)
+                return ret;
+        }
+
+    }
+    else if (bencode_is_string(ben))
+    {
+        const char *str;
+        int len;
+
+        if (0 == bencode_string_value(ben, &str, &len))
+            return -1;
+    }
+    else if (bencode_is_int(ben))
+    {
+        long int val;
+
+        if (0 == bencode_int_value(ben, &val))
+            return -1;
+    }
+    else
+        return -1;
+
+    return 0;
+}
+
+int bencode_validate(char* buf, int len)
+{
+    bencode_t ben;
+    if (0 == len)
+        return 0;
+    bencode_init(&ben, buf, len);
+    return __validate(&ben);
+}

lib/common.c

@@ -51,6 +51,29 @@ tsc(void)
 return 0;
 }
 
+// get microsecond timestamp
+uint64_t
+msts()
+{
+#ifdef _SC_MONOTONIC_CLOCK
+  struct timespec ts;
+  if (sysconf(_SC_MONOTONIC_CLOCK) > 0) {
+    /* A monotonic clock presents */
+    if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0)
+      return (uint64_t)(ts.tv_sec * 1000000 + ts.tv_nsec / 1000);
+    else
+      return 0;
+  }
+  return 0;
+#else
+  struct timeval tv;
+  if (gettimeofday(&tv, NULL) == 0)
+    return (uint64_t)(tv.tv_sec * 1000000 + tv.tv_usec);
+  else
+    return 0;
+#endif
+}
+
 double
 nsts(void)
 {

src/sparsemap.c

@@ -730,15 +730,15 @@ __sm_chunk_scan(__sm_chunk_t *chunk, sm_idx_t start, void (*scanner)(sm_idx_t[],
             continue;
           }
           size_t n = 0;
-          for (size_t b = skip; b < SM_BITS_PER_VECTOR; b++) {
-            buffer[n++] = start + b;
+          for (size_t b = 0; b < SM_BITS_PER_VECTOR; b++) {
+            buffer[n++] = start + ret + b;
           }
           scanner(&buffer[0], n, aux);
           ret += n;
           skip = 0;
         } else {
           for (size_t b = 0; b < SM_BITS_PER_VECTOR; b++) {
-            buffer[b] = start + b;
+            buffer[b] = start + ret + b;
           }
           scanner(&buffer[0], SM_BITS_PER_VECTOR, aux);
           ret += SM_BITS_PER_VECTOR;
@@ -758,14 +758,14 @@ __sm_chunk_scan(__sm_chunk_t *chunk, sm_idx_t start, void (*scanner)(sm_idx_t[],
               continue;
             }
             if (w & ((sm_bitvec_t)1 << b)) {
-              buffer[n++] = start + b;
+              buffer[n++] = start + ret + b;
               ret++;
             }
           }
         } else {
           for (int b = 0; b < SM_BITS_PER_VECTOR; b++) {
             if (w & ((sm_bitvec_t)1 << b)) {
-              buffer[n++] = start + b;
+              buffer[n++] = start + ret + b;
             }
           }
           ret += n;

tests/midl.c

@@ -0,0 +1,417 @@
+/** @defgroup idls	ID List Management
+ *	@{
+ */
+/** A generic unsigned ID number. These were entryIDs in back-bdb.
+ *	Preferably it should have the same size as a pointer.
+ */
+typedef size_t MDB_ID;
+
+/** An IDL is an ID List, a sorted array of IDs. The first
+ * element of the array is a counter for how many actual
+ * IDs are in the list. In the original back-bdb code, IDLs are
+ * sorted in ascending order. For libmdb IDLs are sorted in
+ * descending order.
+ */
+typedef MDB_ID *MDB_IDL;
+
+/* IDL sizes - likely should be even bigger
+ *   limiting factors: sizeof(ID), thread stack size
+ */
+#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
+#define MDB_IDL_DB_SIZE (1 << MDB_IDL_LOGN)
+#define MDB_IDL_UM_SIZE (1 << (MDB_IDL_LOGN + 1))
+
+#define MDB_IDL_DB_MAX (MDB_IDL_DB_SIZE - 1)
+#define MDB_IDL_UM_MAX (MDB_IDL_UM_SIZE - 1)
+
+#define MDB_IDL_SIZEOF(ids) (((ids)[0] + 1) * sizeof(MDB_ID))
+#define MDB_IDL_IS_ZERO(ids) ((ids)[0] == 0)
+#define MDB_IDL_CPY(dst, src) (memcpy(dst, src, MDB_IDL_SIZEOF(src)))
+#define MDB_IDL_FIRST(ids) ((ids)[1])
+#define MDB_IDL_LAST(ids) ((ids)[(ids)[0]])
+
+/** Current max length of an #mdb_midl_alloc()ed IDL */
+#define MDB_IDL_ALLOCLEN(ids) ((ids)[-1])
+
+/** Append ID to IDL. The IDL must be big enough. */
+#define mdb_midl_xappend(idl, id)             \
+  do {                                        \
+    MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \
+    xidl[xlen] = (id);                        \
+  } while (0)
+
+/** Search for an ID in an IDL.
+ * @param[in] ids	The IDL to search.
+ * @param[in] id	The ID to search for.
+ * @return	The index of the first ID greater than or equal to \b id.
+ */
+unsigned mdb_midl_search(MDB_IDL ids, MDB_ID id);
+
+/** Allocate an IDL.
+ * Allocates memory for an IDL of the given size.
+ * @return	IDL on success, NULL on failure.
+ */
+MDB_IDL mdb_midl_alloc(int num);
+
+/** Free an IDL.
+ * @param[in] ids	The IDL to free.
+ */
+void mdb_midl_free(MDB_IDL ids);
+
+/** Shrink an IDL.
+ * Return the IDL to the default size if it has grown larger.
+ * @param[in,out] idp	Address of the IDL to shrink.
+ */
+void mdb_midl_shrink(MDB_IDL *idp);
+
+/** Shrink an IDL to a specific size.
+ * Resize the IDL to \b size if it is larger.
+ * @param[in,out] idp	Address of the IDL to shrink.
+ * @param[in] size	Capacity to have once resized.
+ */
+void mdb_midl_shrink(MDB_IDL *idp);
+
+/** Make room for num additional elements in an IDL.
+ * @param[in,out] idp	Address of the IDL.
+ * @param[in] num	Number of elements to make room for.
+ * @return	0 on success, ENOMEM on failure.
+ */
+int mdb_midl_need(MDB_IDL *idp, unsigned num);
+
+/** Append an ID onto an IDL.
+ * @param[in,out] idp	Address of the IDL to append to.
+ * @param[in] id	The ID to append.
+ * @return	0 on success, ENOMEM if the IDL is too large.
+ */
+int mdb_midl_append(MDB_IDL *idp, MDB_ID id);
+
+/** Append an IDL onto an IDL.
+ * @param[in,out] idp	Address of the IDL to append to.
+ * @param[in] app	The IDL to append.
+ * @return	0 on success, ENOMEM if the IDL is too large.
+ */
+int mdb_midl_append_list(MDB_IDL *idp, MDB_IDL app);
+
+/** Append an ID range onto an IDL.
+ * @param[in,out] idp	Address of the IDL to append to.
+ * @param[in] id	The lowest ID to append.
+ * @param[in] n		Number of IDs to append.
+ * @return	0 on success, ENOMEM if the IDL is too large.
+ */
+int mdb_midl_append_range(MDB_IDL *idp, MDB_ID id, unsigned n);
+
+/** Merge an IDL onto an IDL. The destination IDL must be big enough.
+ * @param[in] idl	The IDL to merge into.
+ * @param[in] merge	The IDL to merge.
+ */
+void mdb_midl_xmerge(MDB_IDL idl, MDB_IDL merge);
+
+/** Sort an IDL.
+ * @param[in,out] ids	The IDL to sort.
+ */
+void mdb_midl_sort(MDB_IDL ids);
+
+/* midl.c ------------------------------------------------------------------ */
+/** @defgroup idls	ID List Management
+ *	@{
+ */
+#define CMP(x, y) ((x) < (y) ? -1 : (x) > (y))
+
+unsigned
+mdb_midl_search(MDB_IDL ids, MDB_ID id)
+{
+  /*
+   * binary search of id in ids
+   * if found, returns position of id
+   * if not found, returns first position greater than id
+   */
+  unsigned base = 0;
+  unsigned cursor = 1;
+  int val = 0;
+  unsigned n = ids[0];
+
+  while (0 < n) {
+    unsigned pivot = n >> 1;
+    cursor = base + pivot + 1;
+    val = CMP(ids[cursor], id);
+
+    if (val < 0) {
+      n = pivot;
+
+    } else if (val > 0) {
+      base = cursor;
+      n -= pivot + 1;
+
+    } else {
+      return cursor;
+    }
+  }
+
+  if (val > 0) {
+    ++cursor;
+  }
+  return cursor;
+}
+
+int
+mdb_midl_insert(MDB_IDL ids, MDB_ID id)
+{
+  unsigned x, i;
+
+  x = mdb_midl_search(ids, id);
+  assert(x > 0);
+
+  if (x < 1) {
+    /* internal error */
+    return -2;
+  }
+
+  if (x <= ids[0] && ids[x] == id) {
+    /* duplicate */
+    assert(0);
+    return -1;
+  }
+
+  if (++ids[0] >= MDB_IDL_DB_MAX) {
+    /* no room */
+    --ids[0];
+    return -2;
+
+  } else {
+    /* insert id */
+    for (i = ids[0]; i > x; i--)
+      ids[i] = ids[i - 1];
+    ids[x] = id;
+  }
+
+  return 0;
+}
+
+inline void
+mdb_midl_pop_n(MDB_IDL ids, unsigned n)
+{
+  ids[0] = ids[0] - n;
+}
+
+void
+mdb_midl_remove_at(MDB_IDL ids, unsigned idx)
+{
+  for (int i = idx - 1; idx < ids[0] - 1;)
+    ids[++i] = ids[++idx];
+  ids[0] = ids[0] - 1;
+}
+
+void
+mdb_midl_remove(MDB_IDL ids, MDB_ID id)
+{
+  unsigned idx = mdb_midl_search(ids, id);
+  if (idx <= ids[0] && ids[idx] == id)
+    mdb_midl_remove_at(ids, idx);
+}
+
+MDB_IDL
+mdb_midl_alloc(int num)
+{
+  MDB_IDL ids = malloc((num + 2) * sizeof(MDB_ID));
+  if (ids) {
+    *ids++ = num;
+    *ids = 0;
+  }
+  return ids;
+}
+
+void
+mdb_midl_free(MDB_IDL ids)
+{
+  if (ids)
+    free(ids - 1);
+}
+
+void
+mdb_midl_shrink(MDB_IDL *idp)
+{
+  MDB_IDL ids = *idp;
+  if (*(--ids) > MDB_IDL_UM_MAX && (ids = realloc(ids, (MDB_IDL_UM_MAX + 2) * sizeof(MDB_ID)))) {
+    *ids++ = MDB_IDL_UM_MAX;
+    *idp = ids;
+  }
+}
+
+void
+mdb_midl_shrink_to(MDB_IDL *idp, size_t size)
+{
+  MDB_IDL ids = *idp;
+  if (*(--ids) > size && (ids = realloc(ids, (size + 2) * sizeof(MDB_ID)))) {
+    *ids++ = size;
+    *idp = ids;
+    *idp[0] = *idp[0] > size ? size : *idp[0];
+  }
+}
+
+static int
+mdb_midl_grow(MDB_IDL *idp, int num)
+{
+  MDB_IDL idn = *idp - 1;
+  /* grow it */
+  idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID));
+  if (!idn)
+    return ENOMEM;
+  *idn++ += num;
+  *idp = idn;
+  return 0;
+}
+
+int
+mdb_midl_need(MDB_IDL *idp, unsigned num)
+{
+  MDB_IDL ids = *idp;
+  num += ids[0];
+  if (num > ids[-1]) {
+    num = (num + num / 4 + (256 + 2)) & -256;
+    if (!(ids = realloc(ids - 1, num * sizeof(MDB_ID))))
+      return ENOMEM;
+    *ids++ = num - 2;
+    *idp = ids;
+  }
+  return 0;
+}
+
+int
+mdb_midl_append(MDB_IDL *idp, MDB_ID id)
+{
+  MDB_IDL ids = *idp;
+  /* Too big? */
+  if (ids[0] >= ids[-1]) {
+    if (mdb_midl_grow(idp, MDB_IDL_UM_MAX))
+      return ENOMEM;
+    ids = *idp;
+  }
+  ids[0]++;
+  ids[ids[0]] = id;
+  return 0;
+}
+
+int
+mdb_midl_append_list(MDB_IDL *idp, MDB_IDL app)
+{
+  MDB_IDL ids = *idp;
+  /* Too big? */
+  if (ids[0] + app[0] >= ids[-1]) {
+    if (mdb_midl_grow(idp, app[0]))
+      return ENOMEM;
+    ids = *idp;
+  }
+  memcpy(&ids[ids[0] + 1], &app[1], app[0] * sizeof(MDB_ID));
+  ids[0] += app[0];
+  return 0;
+}
+
+int
+mdb_midl_append_range(MDB_IDL *idp, MDB_ID id, unsigned n)
+{
+  MDB_ID *ids = *idp, len = ids[0];
+  /* Too big? */
+  if (len + n > ids[-1]) {
+    if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX))
+      return ENOMEM;
+    ids = *idp;
+  }
+  ids[0] = len + n;
+  ids += len;
+  while (n)
+    ids[n--] = id++;
+  return 0;
+}
+
+void
+mdb_midl_xmerge(MDB_IDL idl, MDB_IDL merge)
+{
+  MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i + j, total = k;
+  idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */
+  old_id = idl[j];
+  while (i) {
+    merge_id = merge[i--];
+    for (; old_id < merge_id; old_id = idl[--j])
+      idl[k--] = old_id;
+    idl[k--] = merge_id;
+  }
+  idl[0] = total;
+}
+
+/* Quicksort + Insertion sort for small arrays */
+
+#define SMALL 8
+#define MIDL_SWAP(a, b) \
+  {                     \
+    itmp = (a);         \
+    (a) = (b);          \
+    (b) = itmp;         \
+  }
+
+void
+mdb_midl_sort(MDB_IDL ids)
+{
+  /* Max possible depth of int-indexed tree * 2 items/level */
+  int istack[sizeof(int) * CHAR_BIT * 2];
+  int i, j, k, l, ir, jstack;
+  MDB_ID a, itmp;
+
+  ir = (int)ids[0];
+  l = 1;
+  jstack = 0;
+  for (;;) {
+    if (ir - l < SMALL) { /* Insertion sort */
+      for (j = l + 1; j <= ir; j++) {
+        a = ids[j];
+        for (i = j - 1; i >= 1; i--) {
+          if (ids[i] >= a)
+            break;
+          ids[i + 1] = ids[i];
+        }
+        ids[i + 1] = a;
+      }
+      if (jstack == 0)
+        break;
+      ir = istack[jstack--];
+      l = istack[jstack--];
+    } else {
+      k = (l + ir) >> 1; /* Choose median of left, center, right */
+      MIDL_SWAP(ids[k], ids[l + 1]);
+      if (ids[l] < ids[ir]) {
+        MIDL_SWAP(ids[l], ids[ir]);
+      }
+      if (ids[l + 1] < ids[ir]) {
+        MIDL_SWAP(ids[l + 1], ids[ir]);
+      }
+      if (ids[l] < ids[l + 1]) {
+        MIDL_SWAP(ids[l], ids[l + 1]);
+      }
+      i = l + 1;
+      j = ir;
+      a = ids[l + 1];
+      for (;;) {
+        do
+          i++;
+        while (ids[i] > a);
+        do
+          j--;
+        while (ids[j] < a);
+        if (j < i)
+          break;
+        MIDL_SWAP(ids[i], ids[j]);
+      }
+      ids[l + 1] = ids[j];
+      ids[j] = a;
+      jstack += 2;
+      if (ir - i + 1 >= j - l) {
+        istack[jstack] = ir;
+        istack[jstack - 1] = i;
+        ir = j - 1;
+      } else {
+        istack[jstack] = j - 1;
+        istack[jstack - 1] = l;
+        l = i;
+      }
+    }
+  }
+}