merged commit 831abf2 from illumos-gate

7236 libumem should be able to abort() when an allocation fails

.gitignore

@@ -7,6 +7,7 @@ INSTALL
 Makefile.in
 aclocal.m4
 autom4te.cache
+compile
 config.guess
 config.h.in
 config.sub
@@ -23,6 +24,7 @@ config.log
 config.status
 libtool
 stamp-h1
+test-driver
 umem.spec
 umem_test
 umem_test1

TODO

@@ -17,3 +17,5 @@ To-do List for the Linux port of umem
 * doxygen'ate the headers/code, to produce reference docs.
 
 * HAVE_DOT in Doxyfile.in should be detected by configure.
+
+* option to not use brk() as it's no longer supported on OSX as of 10.10

amd64/umem_genasm.c

@@ -0,0 +1,602 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2013 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you find the blocks of assembly that follow confusing and
+ * you're questioning why they exist, please go read section 8 of the umem.c big
+ * theory statement. Next familiarize yourself with the malloc and free
+ * implementations in libumem's malloc.c.
+ *
+ * What follows is the amd64 implementation of the thread caching automatic
+ * assembly generation. The amd64 calling conventions are documented in the
+ * 64-bit System V ABI. For our purposes what matters is that our first argument
+ * will come in rdi. Our functions have to preserve rbp, rbx, and r12->r15. We
+ * are free to do whatever we want with rax, rcx, rdx, rsi, rdi, and r8->r11.
+ *
+ * For both our implementation of malloc and free we only use the registers we
+ * don't have to preserve.
+ *
+ * Malloc register usage:
+ * 	o. rdi: Original size to malloc. This never changes and is preserved.
+ * 	o. rsi: Adjusted malloc size for malloc_data_tag(s).
+ * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. rdx: Pointer to the tmem_t array of roots
+ * 	o. r8:  Size of the cache
+ * 	o. r9:  Scratch register
+ *
+ * Free register usage:
+ *	o. rdi: Original buffer to free. This never changes and is preserved.
+ *	o. rax: The actual buffer, adjusted for the hidden malloc_data_t(s).
+ * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. rdx: Pointer to the tmem_t array of roots
+ * 	o. r8:  Size of the cache
+ * 	o. r9:  Scratch register
+ *
+ * Once we determine what cache we are using, we increment %rdx to the
+ * appropriate offset and set %r8 with the size of the cache. This means that
+ * when we break out to the normal buffer allocation point %rdx contains the
+ * head of the linked list and %r8 is the amount that we have to adjust the
+ * thread's cached amount by.
+ *
+ * Each block of assembly has psuedocode that describes its purpose.
+ */
+
+#include <atomic.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+const int umem_genasm_supported = 1;
+static uintptr_t umem_genasm_mptr = (uintptr_t)&_malloc;
+static size_t umem_genasm_msize = 576;
+static uintptr_t umem_genasm_fptr = (uintptr_t)&_free;
+static size_t umem_genasm_fsize = 576;
+static uintptr_t umem_genasm_omptr = (uintptr_t)umem_malloc;
+static uintptr_t umem_genasm_ofptr = (uintptr_t)umem_malloc_free;
+
+#define	UMEM_GENASM_MAX64	(UINT32_MAX / sizeof (uintptr_t))
+#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
+#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
+#define	MULTINOP	0x0000441f0f
+
+/*
+ * void *ptcmalloc(size_t orig_size);
+ *
+ * size_t size = orig_size + 8;
+ * if (size > UMEM_SECOND_ALIGN)
+ * 	size += 8;
+ *
+ * if (size < orig_size)
+ * 	goto tomalloc;		! This is overflow
+ *
+ * if (size > cache_max)
+ * 	goto tomalloc
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_MALINIT_JOUT	0x13
+#define	PTC_MALINIT_MCS	0x1a
+#define	PTC_MALINIT_JOV	0x20
+#define	PTC_MALINIT_SOFF	0x30
+static const uint8_t malinit[] =  {
+	0x48, 0x8d, 0x77, 0x08,		/* leaq 0x8(%rdi),%rsi */
+	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10, %rsi */
+	0x76, 0x04,			/* jbe +0x4 */
+	0x48, 0x8d, 0x77, 0x10,		/* leaq 0x10(%rdi),%rsi */
+	0x48, 0x39, 0xfe,		/* cmpq %rdi,%rsi */
+	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00,	/* jb +errout */
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
+	0x64, 0x48, 0x8b, 0x0c, 0x25,
+	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
+	0x48, 0x81, 0xc1,
+	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
+	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
+};
+
+/*
+ * void ptcfree(void *buf);
+ *
+ * if (buf == NULL)
+ * 	return;
+ *
+ * malloc_data_t *tag = buf;
+ * tag--;
+ * int size = tag->malloc_size;
+ * int tagval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
+ * if (tagval == MALLOC_SECOND_MAGIC) {
+ * 	tag--;
+ * } else if (tagval != MALLOC_MAGIC) {
+ * 	goto tofree;
+ * }
+ *
+ * if (size > cache_max)
+ * 	goto tofree;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_FRINI_JDONE	0x05
+#define	PTC_FRINI_JFREE	0x25
+#define	PTC_FRINI_MCS	0x30
+#define	PTC_FRINI_JOV	0x36
+#define	PTC_FRINI_SOFF	0x46
+static const uint8_t freeinit[] = {
+	0x48, 0x85, 0xff,		/* testq %rdi,%rdi */
+	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* jmp $JDONE (done) */
+	0x8b, 0x77, 0xf8,		/* movl -0x8(%rdi),%esi */
+	0x8b, 0x47, 0xfc,		/* movl -0x4(%rdi),%eax */
+	0x01, 0xf0,			/* addl %esi,%eax */
+	0x3d, 0x00, 0x70, 0xba, 0x16,	/* cmpl $MALLOC_2_MAGIC, %eax */
+	0x75, 0x06,			/* jne +0x6 (checkover) */
+	0x48, 0x8d, 0x47, 0xf0,		/* leaq -0x10(%rdi),%eax */
+	0xeb, 0x0f,			/* jmp +0xf (freebuf) */
+	0x3d, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl $MALLOC_MAGIC, %eax */
+	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jmp +JFREE (goto torfree) */
+	0x48, 0x8d, 0x47, 0xf8,		/* leaq -0x8(%rdi),%rax */
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
+	0x64, 0x48, 0x8b, 0x0c, 0x25,
+	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
+	0x48, 0x81, 0xc1,
+	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
+	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ * } else ...				! goto next cache
+ */
+#define	PTC_INICACHE_CMP	0x03
+#define	PTC_INICACHE_SIZE	0x0c
+#define	PTC_INICACHE_JMP	0x11
+static const uint8_t inicache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x0c,			/* ja +0xc (next cache) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0xe9, 0x00, 0x00, 0x00, 0x00,	/* jmp $JMP (allocbuf) */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else ...				! goto next cache
+ */
+#define	PTC_GENCACHE_CMP	0x03
+#define	PTC_GENCACHE_SIZE	0x0c
+#define	PTC_GENCACHE_NUM	0x13
+#define	PTC_GENCACHE_JMP	0x18
+static const uint8_t gencache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x14,			/* ja +0xc (next cache) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0x48, 0x81, 0xc2,
+	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp +$JMP (allocbuf ) */
+};
+
+/*
+ * else if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else {
+ *	goto tofunc; 			! goto tomalloc if ptcmalloc.
+ * }					! goto tofree if ptcfree.
+ */
+#define	PTC_FINCACHE_CMP	0x03
+#define	PTC_FINCACHE_JMP	0x08
+#define	PTC_FINCACHE_SIZE	0x0c
+#define	PTC_FINCACHE_NUM	0x13
+static const uint8_t fincache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x00,			/* ja +JMP (to real malloc) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0x48, 0x81, 0xc2,
+	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
+
+};
+
+/*
+ * if (*root == NULL)
+ * 	goto tomalloc;
+ *
+ * malloc_data_t *ret = *root;
+ * *root = *(void **)ret;
+ * t->tm_size += csize;
+ * ret->malloc_size = size;
+ *
+ * if (size > UMEM_SECOND_ALIGN) {
+ *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ *	ret += 2;
+ * } else {
+ *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ *	ret += 1;
+ * }
+ *
+ * return ((void *)ret);
+ * tomalloc:
+ * 	return (malloc(orig_size));
+ */
+#define	PTC_MALFINI_ALLABEL	0x00
+#define	PTC_MALFINI_JMLABEL	0x40
+#define	PTC_MALFINI_JMADDR	0x41
+static const uint8_t malfini[] = {
+	0x48, 0x8b, 0x02,		/* movl (%rdx),%rax */
+	0x48, 0x85, 0xc0,		/* testq %rax,%rax */
+	0x74, 0x38,			/* je +0x38 (errout) */
+	0x4c, 0x8b, 0x08,		/* movq (%rax),%r9 */
+	0x4c, 0x89, 0x0a,		/* movq %r9,(%rdx) */
+	0x4c, 0x29, 0x01,		/* subq %rsi,(%rcx) */
+	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10,%rsi */
+	0x76, 0x15,			/* jbe +0x15 */
+	0x41, 0xb9, 0x00, 0x70, 0xba, 0x16, /* movl $MALLOC_MAGIC_2, %r9d */
+	0x89, 0x70, 0x08,		/* movl %r9d,0x8(%rax) */
+	0x41, 0x29, 0xf1,		/* subl %esi, %r9d */
+	0x44, 0x89, 0x48, 0x0c,		/* movl %r9d, 0xc(%rax) */
+	0x48, 0x83, 0xc0, 0x10,		/* addq $0x10, %rax */
+	0xc3,				/* ret */
+	0x41, 0xb9, 0x00, 0xc0, 0x10, 0x3a,	/* movl %MALLOC_MAGIC, %r9d */
+	0x89, 0x30,			/* movl %esi,(%rax) */
+	0x41, 0x29, 0xf1,		/* subl %esi,%r9d */
+	0x44, 0x89, 0x48, 0x04,		/* movl %r9d,0x4(%rax) */
+	0x48, 0x83, 0xc0, 0x08,		/* addq $0x8,%rax */
+	0xc3,				/* ret */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $MALLOC */
+};
+
+/*
+ * if (t->tm_size + csize > umem_ptc_size)
+ * 	goto tofree;
+ *
+ * t->tm_size += csize
+ * *(void **)tag = *root;
+ * *root = tag;
+ * return;
+ * tofree:
+ * 	free(buf);
+ * 	return;
+ */
+#define	PTC_FRFINI_RBUFLABEL	0x00
+#define	PTC_FRFINI_CACHEMAX	0x09
+#define	PTC_FRFINI_DONELABEL	0x1b
+#define	PTC_FRFINI_JFLABEL	0x1c
+#define	PTC_FRFINI_JFADDR	0x1d
+static const uint8_t freefini[] = {
+	0x4c, 0x8b, 0x09,		/* movq (%rcx),%r9 */
+	0x4d, 0x01, 0xc1,		/* addq %r8, %r9 */
+	0x49, 0x81, 0xf9,
+	0x00, 0x00, 0x00, 0x00,		/* cmpl $THR_CACHE_MAX, %r9 */
+	0x77, 0x0d,			/* jae +0xd (torfree) */
+	0x4c, 0x01, 0x01,		/* addq %r8,(%rcx) */
+	0x4c, 0x8b, 0x0a,		/* movq (%rdx),%r9 */
+	0x4c, 0x89, 0x08,		/* movq %r9,(%rax) */
+	0x48, 0x89, 0x02,		/* movq %rax,(%rdx) */
+	0xc3,				/* ret */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp free */
+};
+
+/*
+ * Construct the initial part of malloc. off contains the offset from curthread
+ * to the root of the tmem structure. ep is the address of the label to error
+ * and jump to free. csize is the size of the largest umem_cache in ptcumem.
+ */
+static int
+genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
+{
+	uint32_t addr;
+
+	bcopy(malinit, bp, sizeof (malinit));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
+	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
+	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
+	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));
+
+	return (sizeof (malinit));
+}
+
+static int
+genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mcs)
+{
+	uint32_t addr;
+
+	bcopy(freeinit, bp, sizeof (freeinit));
+	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
+	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
+	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
+	bcopy(&mcs, bp + PTC_FRINI_MCS, sizeof (mcs));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
+	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
+	return (sizeof (freeinit));
+}
+
+
+/*
+ * Create the initial cache entry of the specified size. The value of ap tells
+ * us what the address of the label to try and allocate a buffer. This value is
+ * an offset from the current base to that value.
+ */
+static int
+genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+
+	bcopy(inicache, bp, sizeof (inicache));
+	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
+	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
+	ASSERT(addr != 0);
+	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));
+
+	return (sizeof (inicache));
+}
+
+static int
+genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+	uint32_t coff;
+
+	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
+	ASSERT(num != 0);
+	bcopy(gencache, bp, sizeof (gencache));
+	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
+	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
+	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));
+
+	return (sizeof (gencache));
+}
+
+static int
+genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
+{
+	uint8_t eap;
+	uint32_t coff;
+
+	ASSERT(ep <= 0xff && ep > 7);
+	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
+	bcopy(fincache, bp, sizeof (fincache));
+	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_FINCACHE_NUM, sizeof (coff));
+	eap = ep - PTC_FINCACHE_JMP - 1;
+	bcopy(&eap, bp + PTC_FINCACHE_JMP, sizeof (eap));
+
+	return (sizeof (fincache));
+}
+
+static int
+genasm_malfini(uint8_t *bp, uintptr_t mptr)
+{
+	uint32_t addr;
+
+	bcopy(malfini, bp, sizeof (malfini));
+	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
+	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));
+
+	return (sizeof (malfini));
+}
+
+static int
+genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
+{
+	uint32_t addr;
+
+	bcopy(freefini, bp, sizeof (freefini));
+	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
+	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
+	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));
+
+	return (sizeof (freefini));
+}
+
+/*
+ * The malloc inline assembly is constructed as follows:
+ *
+ * o Malloc prologue assembly
+ * o Generic first-cache check
+ * o n Generic cache checks (where n = _tmem_get_entries() - 2)
+ * o Generic last-cache check
+ * o Malloc epilogue assembly
+ *
+ * Generally there are at least three caches. When there is only one cache we
+ * only use the generic last-cache. In the case where there are two caches, we
+ * just leave out the middle ones.
+ */
+static int
+genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	int ii, off;
+	uint8_t *bp;
+	size_t total;
+	uint32_t allocoff, erroff;
+
+	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);
+
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
+	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;
+
+	bp = base;
+
+	off = genasm_malinit(bp, umem_tmem_off, erroff,
+	    umem_alloc_sizes[nents-1]);
+	bp += off;
+	allocoff -= off;
+	erroff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_malfini(bp, umem_genasm_omptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+static int
+genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	uint8_t *bp;
+	int ii, off;
+	size_t total;
+	uint32_t rbufoff, retoff, erroff;
+
+	/* Assume that nents has already been audited for us */
+	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
+	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
+	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);
+
+	bp = base;
+
+	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
+	    umem_alloc_sizes[nents - 1]);
+	bp += off;
+	erroff -= off;
+	rbufoff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
+		bp += off;
+		erroff -= off;
+		rbufoff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
+		bp += off;
+		rbufoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+/*ARGSUSED*/
+int
+umem_genasm(int *cp, umem_cache_t **caches, int nc)
+{
+	int nents, i;
+	uint8_t *mptr;
+	uint8_t *fptr;
+	uint64_t v, *vptr;
+
+	mptr = (void *)((uintptr_t)umem_genasm_mptr + 5);
+	fptr = (void *)((uintptr_t)umem_genasm_fptr + 5);
+	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
+	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
+		return (1);
+
+	/*
+	 * The total number of caches that we can service is the minimum of:
+	 *  o the amount supported by libc
+	 *  o the total number of umem caches
+	 *  o we use a single byte addl, so it's MAX_UINT32 / sizeof (uintptr_t)
+	 *    For 64-bit, this is MAX_UINT32 >> 3, a lot.
+	 */
+	nents = _tmem_get_nentries();
+
+	if (UMEM_GENASM_MAX64 < nents)
+		nents = UMEM_GENASM_MAX64;
+
+	if (nc < nents)
+		nents = nc;
+
+	/* Based on our constraints, this is not an error */
+	if (nents == 0 || umem_ptc_size == 0)
+		return (0);
+
+	/* Take into account the jump */
+	if (genasm_malloc(mptr, umem_genasm_msize, nents, cp) != 0)
+		return (1);
+
+	if (genasm_free(fptr, umem_genasm_fsize, nents, cp) != 0)
+		return (1);
+
+
+	/* nop out the jump with a multibyte jump */
+	vptr = (void *)umem_genasm_mptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+	vptr = (void *)umem_genasm_fptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+
+	for (i = 0; i < nents; i++)
+		caches[i]->cache_flags |= UMF_PTC;
+
+	return (0);
+}

configure.ac

@@ -1,6 +1,8 @@
 AC_INIT([umem], [1.0.2], [], [umem])
 AM_INIT_AUTOMAKE([dist-bzip2])
 
+AC_CONFIG_MACRO_DIRS([m4])
+
 AC_PROG_CC
 AM_PROG_AS
 AC_PROG_LIBTOOL
@@ -8,8 +10,8 @@ AC_PROG_LIBTOOL
 AC_C_INLINE
 
 AC_MSG_CHECKING([whether pthread_mutex_t is larger than 24 bytes])
-AC_TRY_RUN( 
-  [   
+AC_TRY_RUN(
+  [
 #include <pthread.h>
 int main(void){return (sizeof(pthread_mutex_t) > 24);}
   ],
@@ -19,7 +21,7 @@ int main(void){return (sizeof(pthread_mutex_t) > 24);}
     AC_DEFINE(UMEM_PTHREAD_MUTEX_TOO_BIG, [1], [need bigger cache])
     AC_MSG_WARN([*** increasing umem cpu cache size to compensate.])
   ]
-) 
+)
 
 AC_CHECK_LIB(dl,dlopen)
 

envvar.c

@@ -24,7 +24,7 @@
  * Use is subject to license terms.
  *
  * Portions Copyright 2012 Joyent, Inc. All rights reserved.
- *
+ * Portions Copyright 2015 by Delphix. All rights reserved.
  * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
@@ -181,7 +181,10 @@ static umem_env_item_t umem_options_items[] = {
 	},
 #endif
 #endif
-
+	{ "perthread_cache",	"Evolving",	ITEM_SIZE,
+		"Size (in bytes) of per-thread allocation cache",
+		NULL, 0, NULL, &umem_ptc_size
+	},
 	{ NULL, "-- end of UMEM_OPTIONS --",	ITEM_INVALID }
 };
 
@@ -250,6 +253,10 @@ static umem_env_item_t umem_debug_items[] = {
 		"Enables writing all logged messages to stderr",
 		&umem_output,	2
 	},
+	{ "checknull",		"Private",	ITEM_FLAG,
+		"Abort if an allocation would return null",
+		&umem_flags,	UMF_CHECKNULL
+	},
 
 	{ NULL, "-- end of UMEM_DEBUG --",	ITEM_INVALID }
 };
@@ -558,6 +565,9 @@ process_item(const umem_env_item_t *item, const char *item_arg)
 	case ITEM_SIZE:
 		arg_required = 1;
 		break;
+	case ITEM_INVALID:
+	default:
+		break;
 	}
 
 	switch (item->item_type) {

i386/asm_subr.s

@@ -0,0 +1,115 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/asm_linkage.h>
+
+#define	NOP4	\
+	nop;	\
+	nop;	\
+	nop;	\
+	nop;
+
+#define NOP16	\
+	NOP4	\
+	NOP4	\
+	NOP4	\
+	NOP4
+
+#define	NOP64	\
+	NOP16	\
+	NOP16	\
+	NOP16	\
+	NOP16
+
+#define	NOP256	\
+	NOP64	\
+	NOP64	\
+	NOP64	\
+	NOP64
+
+#if defined(lint)
+
+void *
+getfp(void)
+{
+	return (NULL);
+}
+
+#ifndef UMEM_STANDALONE
+void
+_breakpoint(void)
+{
+	return;
+}
+#endif
+
+#else	/* lint */
+
+#if defined(__amd64)
+
+	ENTRY(getfp)
+	movq	%rbp, %rax
+	ret
+	SET_SIZE(getfp)
+
+#else	/* __i386 */
+
+	ENTRY(getfp)
+	movl	%ebp, %eax
+	ret
+	SET_SIZE(getfp)
+
+#endif
+
+#ifndef UMEM_STANDALONE
+	ENTRY(_breakpoint)
+	int	$3
+	ret
+	SET_SIZE(_breakpoint)
+#endif
+
+	ENTRY(_malloc)
+	jmp umem_malloc;
+	NOP256
+	NOP256
+#if defined(__amd64)
+	NOP64
+#endif
+	SET_SIZE(_malloc)
+
+	ENTRY(_free)
+	jmp umem_malloc_free;
+	NOP256
+	NOP256
+#if defined(__amd64)
+	NOP64
+#endif
+	SET_SIZE(_free)
+
+	ANSI_PRAGMA_WEAK2(malloc,_malloc,function)
+	ANSI_PRAGMA_WEAK2(free,_free,function)
+	
+#endif	/* lint */

i386/umem_genasm.c

@@ -0,0 +1,595 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you find the blocks of assembly that follow confusing and
+ * you're questioning why they exist, please go read section 8 of the umem.c big
+ * theory statement. Next familiarize yourself with the malloc and free
+ * implementations in libumem's malloc.c.
+ *
+ * What follows is the i386 implementation of the thread caching automatic
+ * assembly generation. With i386 a function only has three registers it's
+ * allowed to change without restoring them: eax, ecx, and edx. All others have
+ * to be preserved. Since the set of registers we have available is so small, we
+ * have to make use of esi, ebx, and edi and save their original values to the
+ * stack.
+ *
+ * Malloc register usage:
+ * 	o. esi: Size of the malloc (passed into us and modified)
+ * 	o. edi: Size of the cache
+ * 	o. eax: Buffer to return
+ * 	o. ebx: Scratch space and temporary values
+ * 	o. ecx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. edx: Pointer to the tmem_t array of roots
+ *
+ * Free register usage:
+ * 	o. esi: Size of the malloc (passed into us and modified)
+ * 	o. edi: Size of the cache
+ * 	o. eax: Buffer to free
+ * 	o. ebx: Scratch space and temporary values
+ * 	o. ecx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. edx: Pointer to the tmem_t array of roots
+ *
+ * Once we determine what cache we are using, we increment %edx to the
+ * appropriate offset and set %edi with the size of the cache. This means that
+ * when we break out to the normal buffer allocation point %edx contains the
+ * head of the linked list and %edi is the amount that we have to adjust the
+ * total amount cached by the thread.
+ *
+ * Each block of assembly has psuedocode that describes its purpose.
+ */
+
+#include <inttypes.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+#include <atomic.h>
+
+const int umem_genasm_supported = 1;
+static uintptr_t umem_genasm_mptr = (uintptr_t)&_malloc;
+static size_t umem_genasm_msize = 512;
+static uintptr_t umem_genasm_fptr = (uintptr_t)&_free;
+static size_t umem_genasm_fsize = 512;
+static uintptr_t umem_genasm_omptr = (uintptr_t)umem_malloc;
+static uintptr_t umem_genasm_ofptr = (uintptr_t)umem_malloc_free;
+/*
+ * The maximum number of caches we can support. We use a single byte addl so
+ * this is 255 (UINT8_MAX) / sizeof (uintptr_t). In this case 63
+ */
+#define	UMEM_GENASM_MAX32	63
+
+#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
+#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
+#define	MULTINOP	0x0000441f0f
+
+/*
+ * void *ptcmalloc(size_t orig_size);
+ *
+ * size_t size = orig_size + 8;
+ *
+ * if (size < orig_size)
+ * 	goto tomalloc;		! This is overflow
+ *
+ * if (size > cache_size)
+ * 	goto tomalloc;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_MALINIT_JOUT	0x0e
+#define	PTC_MALINIT_MCS	0x14
+#define	PTC_MALINIT_JOV	0x1a
+#define	PTC_MALINIT_SOFF	0x27
+static const uint8_t malinit[] = {
+	0x55,					/* pushl %ebp */
+	0x89, 0xe5,				/* movl %esp, %ebp */
+	0x57,					/* pushl %edi */
+	0x56,					/* pushl %esi */
+	0x53,					/* pushl %ebx */
+	0x8b, 0x75, 0x08,			/* movl 0x8(%ebp), %esi */
+	0x83, 0xc6, 0x08,			/* addl $0x8,%esi */
+	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00, 	/* jc +$JMP (errout) */
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +$JMP (errout) */
+	0x65, 0x8b, 0x0d, 0x00, 0x00, 0x00, 0x00, 	/* movl %gs:0x0,%ecx */
+	0x81, 0xc1, 0x00, 0x00,	0x00, 0x00, 	/* addl $OFF, %ecx */
+	0x8d, 0x51, 0x04			/* leal 0x4(%ecx), %edx */
+};
+
+/*
+ * void ptcfree(void *buf);
+ *
+ * if (buf == NULL)
+ * 	return;
+ *
+ * malloc_data_t *tag = buf;
+ * tag--;
+ * int size = tag->malloc_size;
+ * int tagtval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
+ *
+ * if (tagval != MALLOC_MAGIC)
+ * 	goto tofree;
+ *
+ * if (size > cache_max)
+ * 	goto tofree;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_FRINI_JDONE	0x0d
+#define	PTC_FRINI_JFREE	0x23
+#define	PTC_FRINI_MCS	0x29
+#define	PTC_FRINI_JOV	0x2f
+#define	PTC_FRINI_SOFF	0x3c
+static const uint8_t freeinit[] = {
+	0x55,					/* pushl %ebp */
+	0x89, 0xe5,				/* movl %esp, %ebp */
+	0x57,					/* pushl %edi */
+	0x56,					/* pushl %esi */
+	0x53,					/* pushl %ebx */
+	0x8b, 0x45, 0x08,			/* movl 0x8(%ebp), %eax */
+	0x85, 0xc0,				/* testl %eax, %eax */
+	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* je $JDONE (done) */
+	0x83, 0xe8, 0x08,			/* subl $0x8,%eax */
+	0x8b, 0x30,				/* movl (%eax),%esi */
+	0x8b, 0x50, 0x04,			/* movl 0x4(%eax),%edx */
+	0x01, 0xf2,				/* addl %esi,%edx */
+	0x81, 0xfa, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl MAGIC32, %edx */
+	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jne +JFREE (goto freebuf) */
+
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +$JMP (errout) */
+	0x65, 0x8b, 0x0d, 0x00, 0x0, 0x00, 0x00, /* movl %gs:0x0,%ecx */
+	0x81, 0xc1, 0x00, 0x00,	0x00, 0x00,	/* addl $0xOFF, %ecx */
+	0x8d, 0x51, 0x04			/* leal 0x4(%ecx),%edx */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ * } else ...				! goto next cache
+ */
+#define	PTC_INICACHE_CMP	0x02
+#define	PTC_INICACHE_SIZE 0x09
+#define	PTC_INICACHE_JMP	0x0e
+static const uint8_t inicache[] = {
+	0x81, 0xfe, 0xff, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x77, 0x0a,				/* ja +0xa */
+	0xbf, 0xff, 0x00, 0x00, 0x00, 		/* movl sizeof ($C0), %edi */
+	0xe9, 0x00, 0x00, 0x00, 0x00		/* jmp +$JMP (allocbuf) */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else ...				! goto next cache
+ */
+#define	PTC_GENCACHE_CMP	0x02
+#define	PTC_GENCACHE_NUM	0x0a
+#define	PTC_GENCACHE_SIZE 0x0c
+#define	PTC_GENCACHE_JMP	0x11
+static const uint8_t gencache[] = {
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($CACHE), %esi */
+	0x77, 0x0d,				/* ja +0xd (next cache) */
+	0x83, 0xc2, 0x00,			/* addl $4*$ii, %edx */
+	0xbf, 0x00, 0x00, 0x00, 0x00, 		/* movl sizeof ($CACHE), %edi */
+	0xe9, 0x00, 0x00, 0x00, 0x00 		/* jmp +$JMP (allocbuf) */
+};
+
+/*
+ * else if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else {
+ *	goto tofunc; 			! goto tomalloc if ptcmalloc.
+ * }					! goto tofree if ptcfree.
+ */
+#define	PTC_FINCACHE_CMP 0x02
+#define	PTC_FINCACHE_JMP	0x07
+#define	PTC_FINCACHE_NUM 0x0a
+#define	PTC_FINCACHE_SIZE 0x0c
+static const uint8_t fincache[] = {
+	0x81, 0xfe, 0xff, 0x00, 0x00, 0x00,	/* cmpl sizeof ($CLAST), %esi */
+	0x77, 0x00,				/* ja +$JMP (to errout) */
+	0x83, 0xc2, 0x00,			/* addl $4*($NCACHES-1), %edx */
+	0xbf, 0x00, 0x00, 0x00, 0x00, 		/* movl sizeof ($CLAST), %edi */
+};
+
+/*
+ * if (*root == NULL)
+ * 	goto tomalloc;
+ *
+ * malloc_data_t *ret = *root;
+ * *root = *(void **)ret;
+ * t->tm_size += csize;
+ * ret->malloc_size = size;
+ *
+ * ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ * ret++;
+ *
+ * return ((void *)ret);
+ * tomalloc:
+ * 	return (malloc(orig_size));
+ */
+#define	PTC_MALFINI_ALLABEL	0x00
+#define	PTC_MALFINI_JMLABEL	0x20
+#define	PTC_MALFINI_JMADDR	0x25
+static const uint8_t malfini[] = {
+	/* allocbuf: */
+	0x8b, 0x02,			/* movl (%edx), %eax */
+	0x85, 0xc0,			/* testl %eax, %eax */
+	0x74, 0x1a,			/* je +0x1a (errout) */
+	0x8b, 0x18,			/* movl (%eax), %esi */
+	0x89, 0x1a,			/* movl %esi, (%edx) */
+	0x29, 0x39,			/* subl %edi, (%ecx) */
+	0x89, 0x30,			/* movl %esi, ($eax) */
+	0xba, 0x00, 0xc0, 0x10, 0x3a,	/* movl $0x3a10c000,%edx */
+	0x29, 0xf2,			/* subl %esi, %edx */
+	0x89, 0x50, 0x04,		/* movl %edx, 0x4(%eax) */
+	0x83, 0xc0, 0x08,		/* addl %0x8, %eax */
+	0x5b,				/* popl %ebx */
+	0x5e,				/* popl %esi */
+	0x5f,				/* popl %edi */
+	0xc9,				/* leave */
+	0xc3,				/* ret */
+	/* errout: */
+	0x5b,				/* popl %ebx */
+	0x5e,				/* popl %esi */
+	0x5f,				/* popl %edi */
+	0xc9,				/* leave */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $malloc */
+};
+
+/*
+ * if (t->tm_size + csize > umem_ptc_size)
+ * 	goto tofree;
+ *
+ * t->tm_size += csize
+ * *(void **)tag = *root;
+ * *root = tag;
+ * return;
+ * tofree:
+ * 	free(buf);
+ * 	return;
+ */
+#define	PTC_FRFINI_RBUFLABEL	0x00
+#define	PTC_FRFINI_CACHEMAX	0x06
+#define	PTC_FRFINI_DONELABEL	0x14
+#define	PTC_FRFINI_JFLABEL	0x19
+#define	PTC_FRFINI_JFADDR	0x1e
+static const uint8_t freefini[] = {
+	/* freebuf: */
+	0x8b, 0x19,				/* movl (%ecx),%ebx */
+	0x01, 0xfb,				/* addl %edi,%ebx */
+	0x81, 0xfb, 0x00, 0x00, 0x00, 0x00, 	/* cmpl maxsize, %ebx */
+	0x73, 0x0d,				/* jae +0xd <tofree> */
+	0x01, 0x39,				/* addl %edi,(%ecx) */
+	0x8b, 0x3a,				/* movl (%edx),%edi */
+	0x89, 0x38,				/* movl %edi,(%eax) */
+	0x89, 0x02,				/* movl %eax,(%edx) */
+	/* done: */
+	0x5b,					/* popl %ebx */
+	0x5e,					/* popl %esi */
+	0x5f,					/* popl %edi */
+	0xc9,					/* leave */
+	0xc3,					/* ret */
+	/* realfree: */
+	0x5b,					/* popl %ebx */
+	0x5e,					/* popl %esi */
+	0x5f,					/* popl %edi */
+	0xc9,					/* leave */
+	0xe9, 0x00, 0x00, 0x00, 0x00		/* jmp free */
+};
+
+/*
+ * Construct the initial part of malloc. off contains the offset from curthread
+ * to the root of the tmem structure. ep is the address of the label to error
+ * and jump to free. csize is the size of the largest umem_cache in ptcumem.
+ */
+static int
+genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
+{
+	uint32_t addr;
+
+	bcopy(malinit, bp, sizeof (malinit));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
+	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
+	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
+	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));
+
+	return (sizeof (malinit));
+}
+
+static int
+genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mc)
+{
+	uint32_t addr;
+
+	bcopy(freeinit, bp, sizeof (freeinit));
+	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
+	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
+	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
+	bcopy(&mc, bp + PTC_FRINI_MCS, sizeof (mc));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
+	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
+	return (sizeof (freeinit));
+}
+
+/*
+ * Create the initial cache entry of the specified size. The value of ap tells
+ * us what the address of the label to try and allocate a buffer. This value is
+ * an offset from the current base to that value.
+ */
+static int
+genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+
+	bcopy(inicache, bp, sizeof (inicache));
+	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
+	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
+	ASSERT(addr != 0);
+	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));
+
+	return (sizeof (inicache));
+}
+
+static int
+genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+	uint8_t	coff;
+
+	ASSERT(256 / PTC_ROOT_SIZE > num);
+	ASSERT(num != 0);
+	bcopy(gencache, bp, sizeof (gencache));
+	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
+	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
+	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));
+
+	return (sizeof (gencache));
+}
+
+static int
+genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
+{
+	uint8_t addr;
+
+	ASSERT(ep <= 0xff && ep > 7);
+	ASSERT(256 / PTC_ROOT_SIZE > num);
+	bcopy(fincache, bp, sizeof (fincache));
+	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
+	addr = num * PTC_ROOT_SIZE;
+	bcopy(&addr, bp + PTC_FINCACHE_NUM, sizeof (addr));
+	addr = ep - PTC_FINCACHE_JMP - 1;
+	bcopy(&addr, bp + PTC_FINCACHE_JMP, sizeof (addr));
+
+	return (sizeof (fincache));
+}
+
+static int
+genasm_malfini(uint8_t *bp, uintptr_t mptr)
+{
+	uint32_t addr;
+
+	bcopy(malfini, bp, sizeof (malfini));
+	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
+	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));
+
+	return (sizeof (malfini));
+}
+
+static int
+genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
+{
+	uint32_t addr;
+
+	bcopy(freefini, bp, sizeof (freefini));
+	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
+	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
+	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));
+
+	return (sizeof (freefini));
+}
+
+/*
+ * The malloc inline assembly is constructed as follows:
+ *
+ * o Malloc prologue assembly
+ * o Generic first-cache check
+ * o n Generic cache checks (where n = _tmem_get_entries() - 2)
+ * o Generic last-cache check
+ * o Malloc epilogue assembly
+ *
+ * Generally there are at least three caches. When there is only one cache we
+ * only use the generic last-cache. In the case where there are two caches, we
+ * just leave out the middle ones.
+ */
+static int
+genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	int ii, off;
+	uint8_t *bp;
+	size_t total;
+	uint32_t allocoff, erroff;
+
+	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);
+
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
+	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;
+
+	bp = base;
+
+	off = genasm_malinit(bp, umem_tmem_off, erroff,
+	    umem_alloc_sizes[nents-1]);
+	bp += off;
+	allocoff -= off;
+	erroff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_malfini(bp, umem_genasm_omptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+static int
+genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	uint8_t *bp;
+	int ii, off;
+	size_t total;
+	uint32_t rbufoff, retoff, erroff;
+
+	/* Assume that nents has already been audited for us */
+	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
+	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
+	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);
+
+	bp = base;
+
+	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
+	    umem_alloc_sizes[nents - 1]);
+	bp += off;
+	erroff -= off;
+	rbufoff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
+		bp += off;
+		erroff -= off;
+		rbufoff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
+		bp += off;
+		rbufoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+int
+umem_genasm(int *alloc_sizes, umem_cache_t **caches, int ncaches)
+{
+	int nents, i;
+	uint8_t *mptr;
+	uint8_t *fptr;
+	uint64_t v, *vptr;
+
+	mptr = (void *)((uintptr_t)umem_genasm_mptr + 5);
+	fptr = (void *)((uintptr_t)umem_genasm_fptr + 5);
+	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
+	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
+		return (1);
+
+	/*
+	 * The total number of caches that we can service is the minimum of:
+	 *  o the amount supported by libc
+	 *  o the total number of umem caches
+	 *  o we use a single byte addl, so it's 255 / sizeof (uintptr_t). For
+	 *    32-bit, this is 63.
+	 */
+	nents = _tmem_get_nentries();
+
+	if (UMEM_GENASM_MAX32 < nents)
+		nents = UMEM_GENASM_MAX32;
+
+	if (ncaches < nents)
+		nents = ncaches;
+
+	/* Based on our constraints, this is not an error */
+	if (nents == 0 || umem_ptc_size == 0)
+		return (0);
+
+	/* Take into account the jump */
+	if (genasm_malloc(mptr, umem_genasm_msize, nents,
+	    alloc_sizes) != 0)
+		return (1);
+
+	if (genasm_free(fptr, umem_genasm_fsize, nents,
+	    alloc_sizes) != 0)
+		return (1);
+
+	/* nop out the jump with a multibyte jump */
+	vptr = (void *)umem_genasm_mptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+	vptr = (void *)umem_genasm_fptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+
+	for (i = 0; i < nents; i++)
+		caches[i]->cache_flags |= UMF_PTC;
+
+	return (0);
+}

linktest_stand.c

@@ -34,6 +34,8 @@
 
 void __umem_assert_failed(void) {}
 void _atomic_add_64(void) {}
+void atomic_add_32_nv(void) {}
+void atomic_swap_64(void) {}
 void _atomic_add_32_nv(void) {}
 void bcopy(void) {}
 void bzero(void) {}

malloc.c

@@ -59,6 +59,15 @@ typedef struct malloc_data {
 	uint32_t malloc_stat; /* = UMEM_MALLOC_ENCODE(state, malloc_size) */
 } malloc_data_t;
 
+/*
+ * Because we do not support ptcumem on non-x86 today, we have to create these
+ * weak aliases.
+ */
+#if !defined(__amd64__) && !defined(__x86_64__)
+#pragma weak malloc = umem_malloc
+#pragma weak free = umem_malloc_free
+#endif /* !_x86 */
+
 void *
 malloc(size_t size_arg)
 {

sparc/asm_subr.s

@@ -0,0 +1,92 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <sys/asm_linkage.h>
+
+#if defined(lint)
+
+void *
+getfp(void)
+{
+	return (NULL);
+}
+
+void
+flush_windows(void)
+{
+
+}
+
+#ifndef UMEM_STANDALONE
+void
+_breakpoint(void)
+{
+	return;
+}
+#endif
+
+#else	/* lint */
+
+	ENTRY(getfp)
+	retl
+	mov	%fp, %o0
+	SET_SIZE(getfp)
+
+#ifdef UMEM_STANDALONE
+#ifdef __sparcv9
+
+	/*
+	 * The caller doesn't need the top window to be flushed, so this
+	 * is sufficient.
+	 */
+	ENTRY(flush_windows)
+	retl
+	flushw
+	SET_SIZE(flush_windows)
+
+#else	/* !__sparcv9 */
+#error	"This file does not provide a pre-v9 standalone flush_windows"
+#endif	/* __sparcv9 */
+
+#else	/* !UMEM_STANDALONE */
+
+	ENTRY(flush_windows)
+	retl
+	ta	0x3
+	SET_SIZE(flush_windows)
+
+#endif	/* UMEM_STANDALONE */
+
+#ifndef UMEM_STANDALONE
+	ENTRY(_breakpoint)
+	retl
+	ta	0x1
+	SET_SIZE(_breakpoint)
+#endif
+
+#endif	/* lint */

sparc/umem_genasm.c

@@ -0,0 +1,43 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you wonder why this seemingly empty file exists, it's because
+ * there is no sparc implementation for ptcumem. Go read libumem's big theory
+ * statement in lib/libumem/common/umem.c, particularly section eight.
+ */
+
+#include <inttypes.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+const int umem_genasm_supported = 0;
+
+/*ARGSUSED*/
+int
+umem_genasm(int *alloc_sizes, umem_cache_t **caches, int ncaches)
+{
+	return (1);
+}

stub_stand.c

@@ -21,11 +21,13 @@
  */
 
 /*
+ * Copyright 2014 Garrett D'Amore <garrett@damore.org>
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
-
-#pragma ident	"@(#)stub_stand.c	1.3	05/06/08 SMI"
+/*
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ */
 
 /*
  * Stubs for the standalone to reduce the dependence on external libraries
@@ -68,6 +70,13 @@ _cond_broadcast(cond_t *cvp)
 	return (0);
 }
 
+/*ARGSUSED*/
+int
+pthread_setcancelstate(int state, int *oldstate)
+{
+	return (0);
+}
+
 thread_t
 _thr_self(void)
 {
@@ -124,3 +133,36 @@ issetugid(void)
 {
 	return (1);
 }
+
+int
+_tmem_get_nentries(void)
+{
+	return (0);
+}
+
+uintptr_t
+_tmem_get_base(void)
+{
+	return (0);
+}
+
+/*ARGSUSED*/
+void
+_tmem_set_cleanup(void (*f)(int, void *))
+{
+}
+
+int
+isspace(int c)
+{
+	switch (c) {
+	case ' ':
+	case '\t':
+	case '\n':
+	case '\r':
+	case '\f':
+	case '\v':
+		return (1);
+	}
+	return (0);
+}

umem_alloc.3

@@ -1,190 +1,166 @@
 '\" te
-.\" CDDL HEADER START
-.\"
-.\" The contents of this file are subject to the terms of the
-.\" Common Development and Distribution License (the "License").  
-.\" You may not use this file except in compliance with the License.
-.\"
-.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-.\" or http://www.opensolaris.org/os/licensing.
-.\" See the License for the specific language governing permissions
-.\" and limitations under the License.
-.\"
-.\" When distributing Covered Code, include this CDDL HEADER in each
-.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-.\" If applicable, add the following below this CDDL HEADER, with the
-.\" fields enclosed by brackets "[]" replaced with your own identifying
-.\" information: Portions Copyright [yyyy] [name of copyright owner]
-.\"
-.\" CDDL HEADER END
-.\" Copyright (c) 2002, Sun Microsystems, Inc. All Rights Reserved.
-.TH umem_alloc 3MALLOC "26 Aug 2002" "SunOS 5.11" "Memory Allocation Library Functions"
+.\" Copyright (c) 2008 Sun Microsystems, Inc. All Rights Reserved.
+.\" Copyright (c) 2012 Joyent, Inc. All Rights Reserved.
+.\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License").  You may not use this file except in compliance with the License.
+.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.  See the License for the specific language governing permissions and limitations under the License.
+.\" When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE.  If applicable, add the following below this CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
+.TH UMEM_ALLOC 3MALLOC "Mar 24, 2008"
 .SH NAME
-umem_alloc, umem_zalloc, umem_free, umem_nofail_callback \- fast, scalable memory allocation
+umem_alloc, umem_zalloc, umem_free, umem_nofail_callback \- fast, scalable
+memory allocation
 .SH SYNOPSIS
 .LP
 .nf
-cc [ \fIflag \&.\|.\|.\fR ] \fIfile\fR\&.\|.\|. \fB-lumem\fR [ \fIlibrary \&.\|.\|.\fR ] 
+cc [ \fIflag \&.\|.\|.\fR ] \fIfile\fR\&.\|.\|. \fB-lumem\fR [ \fIlibrary \&.\|.\|.\fR ]
 #include <umem.h>
 
 \fBvoid *\fR\fBumem_alloc\fR(\fBsize_t\fR \fIsize\fR, \fBint\fR  \fIflags\fR);
 .fi
+
 .LP
 .nf
 \fBvoid *\fR\fBumem_zalloc\fR(\fBsize_t\fR \fIsize\fR, \fBint\fR  \fIflags\fR);
 .fi
+
 .LP
 .nf
 \fBvoid\fR \fBumem_free\fR(\fBvoid *\fR\fIbuf\fR, \fBsize_t\fR \fIsize\fR);
 .fi
+
 .LP
 .nf
 \fBvoid\fR \fBumem_nofail_callback\fR(\fB(int (*\fR\fIcallback\fR)(void));
 .fi
+
 .LP
 .nf
 \fBvoid *\fR\fBmalloc\fR(\fBsize_t\fR \fIsize\fR);
 .fi
+
 .LP
 .nf
 \fBvoid *\fR\fBcalloc\fR(\fBsize_t\fR \fInelem\fR, \fBsize_t\fR \fIelsize\fR);
 .fi
+
 .LP
 .nf
 \fBvoid\fR \fBfree\fR(\fBvoid *\fR\fIptr\fR);
 .fi
+
 .LP
 .nf
 \fBvoid *\fR\fBmemalign\fR(\fBsize_t\fR \fIalignment\fR, \fBsize_t\fR \fIsize\fR);
 .fi
+
 .LP
 .nf
 \fBvoid *\fR\fBrealloc\fR(\fBvoid *\fR\fIptr\fR, \fBsize_t\fR \fIsize\fR);
 .fi
+
 .LP
 .nf
 \fBvoid *\fR\fBvalloc\fR(\fBsize_t\fR \fIsize\fR);
 .fi
 
 .SH DESCRIPTION
-
-.LP
-The \fBumem_alloc()\fR function returns a pointer to a block of \fIsize\fR bytes suitably aligned for any variable type. The initial contents of memory allocated using \fBumem_alloc()\fR is undefined. The \fIflags\fR argument determines
-the behavior of \fBumem_alloc()\fR if it is unable to fulfill the request. The \fIflags\fR argument can take the following values:
 .sp
-
+.LP
+The \fBumem_alloc()\fR function returns a pointer to a block of \fIsize\fR
+bytes suitably aligned for any variable type. The initial contents of memory
+allocated using \fBumem_alloc()\fR is undefined. The \fIflags\fR argument
+determines the behavior of \fBumem_alloc()\fR if it is unable to fulfill the
+request. The \fIflags\fR argument can take the following values:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBUMEM_DEFAULT\fR\fR
 .ad
-.RS 14n
-.rt  
+.RS 16n
 Return \fINULL\fR on failure.
-.sp
-
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBUMEM_NOFAIL\fR\fR
 .ad
-.RS 14n
-.rt  
-Call an optional \fIcallback\fR (set with \fBumem_nofail_callback()\fR) on failure. The \fIcallback\fR takes no arguments and can finish by:
-.sp
-
-.sp
+.RS 16n
+Call an optional \fIcallback\fR (set with \fBumem_nofail_callback()\fR) on
+failure. The \fIcallback\fR takes no arguments and can finish by:
 .RS +4
 .TP
 .ie t \(bu
 .el o
-returning \fBUMEM_CALLBACK_RETRY\fR, in which case the allocation will be retried.  If the allocation fails, the callback will be invoked again.
-.sp
-
+returning \fBUMEM_CALLBACK_RETRY\fR, in which case the allocation will be
+retried.  If the allocation fails, the callback will be invoked again.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-returning \fBUMEM_CALLBACK_EXIT\fR(\fIstatus\fR), in which case 
-\fBexit\fR(2) is invoked with \fIstatus\fR
-as its argument. The \fBexit()\fR function is called only once. If multiple threads return from the \fBUMEM_NOFAIL\fR callback with \fBUMEM_CALLBACK_EXIT\fR(\fIstatus\fR), one will call \fBexit()\fR while the other blocks until \fBexit()\fR terminates the program.
-.sp
-
+returning \fBUMEM_CALLBACK_EXIT\fR(\fIstatus\fR), in which case \fBexit\fR(2)
+is invoked with \fIstatus\fR as its argument. The \fBexit()\fR function is
+called only once. If multiple threads return from the \fBUMEM_NOFAIL\fR
+callback with \fBUMEM_CALLBACK_EXIT\fR(\fIstatus\fR), one will call
+\fBexit()\fR while the other blocks until \fBexit()\fR terminates the program.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-invoking a context-changing function (
-\fBsetcontext\fR(2)) or a non-local jump (
-\fBlongjmp\fR(3C) or 
-\fBsiglongjmp\fR(3C), or ending the current thread of control (
-\fBthr_exit\fR(3C) or 
-\fBpthread_exit\fR(3C). The application is responsible for any necessary cleanup. The state of \fBlibumem\fR remains consistent.
+invoking a context-changing function (\fBsetcontext\fR(2)) or a non-local jump
+(\fBlongjmp\fR(3C) or \fBsiglongjmp\fR(3C), or ending the current thread of
+control (\fBthr_exit\fR(3C) or \fBpthread_exit\fR(3C). The application is
+responsible for any necessary cleanup. The state of \fBlibumem\fR remains
+consistent.
+.RE
+If no callback has been set or the callback has been set to \fINULL\fR,
+\fBumem_alloc\fR(..., \fBUMEM_NOFAIL\fR) behaves as though the callback
+returned \fBUMEM_CALLBACK_EXIT\fR(255).
 .sp
-
+The \fBlibumem\fR library can call callbacks from any place that a
+\fBUMEM_NOFAIL\fR allocation is issued. In multithreaded applications,
+callbacks are expected to perform their own concurrency management.
 .RE
 
-If no callback has been set or the callback has been set to \fINULL\fR, \fBumem_alloc\fR(..., \fBUMEM_NOFAIL\fR) behaves as though the callback returned \fBUMEM_CALLBACK_EXIT\fR(255).
 .sp
-
-.sp
-The \fBlibumem\fR library can call callbacks from any place that a \fBUMEM_NOFAIL\fR allocation is issued. In multithreaded applications, callbacks are expected to perform their own concurrency management.
-.sp
-
-.RE
-
 .LP
-The function call \fBumem_alloc\fR(0, \fIflag\fR) always returns \fINULL\fR. The function call \fBumem_free\fR(\fINULL\fR, 0) is allowed.
+The function call \fBumem_alloc\fR(0, \fIflag\fR) always returns \fINULL\fR.
+The function call \fBumem_free\fR(\fINULL\fR, 0) is allowed.
 .sp
-
 .LP
-The \fBumem_zalloc()\fR function has the same semantics as \fBumem_alloc()\fR, but the block of memory is initialized to zeros before it is returned.
+The \fBumem_zalloc()\fR function has the same semantics as \fBumem_alloc()\fR,
+but the block of memory is initialized to zeros before it is returned.
 .sp
-
 .LP
-The \fBumem_free()\fR function frees blocks previously allocated using \fBumem_alloc()\fR and \fBumem_zalloc()\fR. The buffer address and size must exactly match the original allocation. Memory must not be returned piecemeal.
+The \fBumem_free()\fR function frees blocks previously allocated using
+\fBumem_alloc()\fR and \fBumem_zalloc()\fR. The buffer address and size must
+exactly match the original allocation. Memory must not be returned piecemeal.
 .sp
-
 .LP
-The \fBumem_nofail_callback()\fR function sets the process-wide UMEM_NOFAIL callback. See the description of UMEM_NOFAIL for more information.
+The \fBumem_nofail_callback()\fR function sets the process-wide UMEM_NOFAIL
+callback. See the description of UMEM_NOFAIL for more information.
 .sp
-
 .LP
-The \fBmalloc()\fR, \fBcalloc()\fR, \fBfree()\fR, \fBmemalign()\fR, \fBrealloc()\fR, and \fBvalloc()\fR functions are are as described in 
-\fBmalloc\fR(3C). The \fBlibumem\fR library provides these functions for backwards-compatibility with the standard functions.
-.sp
-
+The \fBmalloc()\fR, \fBcalloc()\fR, \fBfree()\fR, \fBmemalign()\fR,
+\fBrealloc()\fR, and \fBvalloc()\fR functions are as described in
+\fBmalloc\fR(3C). The \fBlibumem\fR library provides these functions for
+backwards-compatibility with the standard functions.
 .SH ENVIRONMENT VARIABLES
-
+.sp
 .LP
-See 
-\fBumem_debug\fR(3MALLOC) for environment variables that effect the debugging features of the \fBlibumem\fR library.
-.sp
-
+See \fBumem_debug\fR(3MALLOC) for environment variables that effect the
+debugging features of the \fBlibumem\fR library.
 .sp
 .ne 2
-.mk
 .na
-\fBUMEM_OPTIONS\fR
+\fB\fBUMEM_OPTIONS\fR\fR
 .ad
-.RS 14n
-.rt  
-Contains a list of comma-separated options.  Unrecognized options are ignored. The options that are supported are:
-.sp
-
+.RS 16n
+Contains a list of comma-separated options.  Unrecognized options are ignored.
+The options that are supported are:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBbackend\fR=\fBsbrk\fR\fR
 .ad
@@ -192,14 +168,49 @@ Contains a list of comma-separated options.  Unrecognized options are ignored. T
 .na
 \fB\fBbackend\fR=\fBmmap\fR\fR
 .ad
-.RS 14n
-.rt  
-Set the underlying function used to allocate memory. This option can be set to \fBsbrk\fR (the default) for an 
-\fBsbrk\fR(2)-based source or \fBmmap\fR for an 
-\fBmmap\fR(2)-based
-source. If set to a value that is not supported, \fBsbrk\fR will be used.
+.RS 16n
+Set the underlying function used to allocate memory. This option can be set to
+\fBsbrk\fR (the default) for an \fBsbrk\fR(2)-based source or \fBmmap\fR for an
+\fBmmap\fR(2)-based source. If set to a value that is not supported, \fBsbrk\fR
+will be used.
+.RE
 .sp
+.ne 2
+.na
+\fB\fBperthread_cache\fR=\fBsize\fR\fR
+.ad
+.RS 16n
+libumem allows for each thread to cache recently freed small allocations for
+future allocations. The size argument, which accepts k, m, g, and t, suffixes
+denotes the maximum amount of memory each thread can use for this purpose. The
+default amount used is 1 MB. Any buffers in the per-thread cache are freed when
+the thread exits. The efficacy of the per-thread cache can be determined with
+the \fB::umastat\fR \fBmdb\fR(1) \fIdcmd\fR debugger command.
+.RE
 
+.ne 2
+.na
+\fB\fBallocator\fR=\fBbest\fR\fR
+.ad
+.br
+.na
+\fB\fBallocator\fR=\fBfirst\fR\fR
+.ad
+.br
+.na
+\fB\fBallocator\fR=\fBinstant\fR\fR
+.ad
+.br
+.na
+\fB\fBallocator\fR=\fBnext\fR\fR
+.ad
+.RS 16n
+Set the underlying allocation strategy. The \fBbest\fR fit strategy tells
+libumem to use the smallest free segment possible. The \fBinstant\fR fit
+strategy approximates the best fit strategy in constant cpu time. The
+\fBfirst\fR fit strategy takes the first free segment that can honor the
+allocation. The \fBnext\fR fit strategy uses the next free segment after the
+previously allocated one.
 .RE
 
 .RE
@@ -207,8 +218,7 @@ source. If set to a value that is not supported, \fBsbrk\fR will be used.
 .SH EXAMPLES
 .LP
 \fBExample 1 \fRUsing the \fBumem_alloc()\fR function.
-
-.LP
+.sp
 .in +2
 .nf
 #include <stdio.h>
@@ -217,8 +227,8 @@ source. If set to a value that is not supported, \fBsbrk\fR will be used.
 char *buf = umem_alloc(1024, UMEM_DEFAULT);
 
 if (buf == NULL) {
-    fprintf(stderr, "out of memory\en");
-         return (1);
+     fprintf(stderr, "out of memory\en");
+          return (1);
 }
 /* cannot assume anything about buf's contents */
 \&...
@@ -226,10 +236,10 @@ umem_free(buf, 1024);
 \&...
 .fi
 .in -2
-.LP
-\fBExample 2 \fRUsing the \fBumem_zalloc()\fR function
 
 .LP
+\fBExample 2 \fRUsing the \fBumem_zalloc()\fR function
+.sp
 .in +2
 .nf
 #include <stdio.h>
@@ -238,8 +248,8 @@ umem_free(buf, 1024);
 char *buf = umem_zalloc(1024, UMEM_DEFAULT);
 
 if (buf == NULL) {
-   fprintf(stderr, "out of memory\en");
-        return (1);
+    fprintf(stderr, "out of memory\en");
+         return (1);
 }
 /* buf contains zeros */
 \&...
@@ -247,10 +257,10 @@ umem_free(buf, 1024);
 \&...
 .fi
 .in -2
-.LP
-\fBExample 3 \fRUsing UMEM_NOFAIL
 
 .LP
+\fBExample 3 \fRUsing UMEM_NOFAIL
+.sp
 .in +2
 .nf
 #include <stdlib.h>
@@ -258,14 +268,14 @@ umem_free(buf, 1024);
 #include <umem.h>
 
 /*
-* Note that the allocation code below does not have to
-* check for umem_alloc() returning NULL
-*/
+ * Note that the allocation code below does not have to
+ * check for umem_alloc() returning NULL
+ */
 int
 my_failure_handler(void)
 {
-        (void) fprintf(stderr, "out of memory\en");
-        return (UMEM_CALLBACK_EXIT(255));
+         (void) fprintf(stderr, "out of memory\en");
+         return (UMEM_CALLBACK_EXIT(255));
 }
 \&...
 umem_nofail_callback(my_failure_handler);
@@ -274,17 +284,17 @@ int i;
 char *buf[100];
 
 for (i = 0; i < 100; i++)
-        buf[i] = umem_alloc(1024 * 1024, UMEM_NOFAIL);
+         buf[i] = umem_alloc(1024 * 1024, UMEM_NOFAIL);
 \&...
 for (i = 0; i < 100; i++)
-   umem_free(buf[i], 1024 * 1024);
+    umem_free(buf[i], 1024 * 1024);
 \&...
 .fi
 .in -2
-.LP
-\fBExample 4 \fRUsing UMEM_NOFAIL in a multithreaded application
 
 .LP
+\fBExample 4 \fRUsing UMEM_NOFAIL in a multithreaded application
+.sp
 .in +2
 .nf
 #define _REENTRANT
@@ -295,27 +305,27 @@ for (i = 0; i < 100; i++)
 void *
 start_func(void *the_arg)
 {
-         int *info = (int *)the_arg;
-         char *buf = umem_alloc(1024 * 1024, UMEM_NOFAIL);
+          int *info = (int *)the_arg;
+          char *buf = umem_alloc(1024 * 1024, UMEM_NOFAIL);
 
-         /* does not need to check for buf == NULL */
-         buf[0] = 0;
-         ...
-         /*
-          * if there were other UMEM_NOFAIL allocations,
-          * we would need to arrange for buf to be
-          * umem_free()ed upon failure.
-          */
-         ...
-         umem_free(buf, 1024 * 1024);
-         return (the_arg);
+          /* does not need to check for buf == NULL */
+          buf[0] = 0;
+          ...
+          /*
+           * if there were other UMEM_NOFAIL allocations,
+           * we would need to arrange for buf to be
+           * umem_free()ed upon failure.
+           */
+          ...
+          umem_free(buf, 1024 * 1024);
+          return (the_arg);
 }
 \&...
 int
 my_failure_handler(void)
 {
-        /* terminate the current thread with status NULL */
-        thr_exit(NULL);
+         /* terminate the current thread with status NULL */
+         thr_exit(NULL);
 }
 \&...
 umem_nofail_callback(my_failure_handler);
@@ -326,200 +336,137 @@ thread_t tid;
 void *status;
 
 (void) thr_create(NULL, NULL, start_func, &my_arg, 0,
-   NULL);
+    NULL);
 \&...
 while (thr_join(0, &tid, &status) != 0)
-         ;
+          ;
 
 if (status == NULL) {
-   (void) fprintf(stderr, "thread %d ran out of memory\en",
-            tid);
+    (void) fprintf(stderr, "thread %d ran out of memory\en",
+             tid);
 }
 \&...
 .fi
 .in -2
 
 .SH ATTRIBUTES
-
-.LP
-See 
-\fBattributes\fR(5) for descriptions of the following attributes:
 .sp
-
 .LP
+See \fBattributes\fR(5) for descriptions of the following attributes:
+.sp
 
 .sp
 .TS
-tab() box;
-cw(2.75i) |cw(2.75i) 
-lw(2.75i) |lw(2.75i) 
-.
-ATTRIBUTE TYPEATTRIBUTE VALUE
+box;
+c | c
+l | l .
+ATTRIBUTE TYPE	ATTRIBUTE VALUE
 _
-Interface StabilitySee below.
+Interface Stability	Committed
 _
-MT-LevelMT-Safe
+MT-Level	MT-Safe
+_
+Standard	See below.
 .TE
 
-.LP
-The \fBmalloc()\fR, \fBcalloc()\fR, \fBfree()\fR, \fBrealloc()\fR, and \fBvalloc()\fR functions are Standard. The \fBmemalign()\fR function is Stable. The \fBumem_alloc()\fR, \fBumem_zalloc()\fR, \fBumem_free()\fR, and \fBumem_nofail_callback()\fR functions are Evolving.
 .sp
-
-.SH SEE ALSO
-
 .LP
-
-\fBexit\fR(2), 
-\fBmmap\fR(2), 
-\fBsbrk\fR(2), 
-\fBbsdmalloc\fR(3MALLOC), 
-\fBlibumem\fR(3LIB), 
-\fBlongjmp\fR(3C), 
-\fBmalloc\fR(3C), 
-\fBmalloc\fR(3MALLOC), 
-\fBmapmalloc\fR(3MALLOC), 
-\fBpthread_exit\fR(3C), 
-\fBthr_exit\fR(3C), 
-\fBumem_cache_create\fR(3MALLOC), 
-\fBumem_debug\fR(3MALLOC), 
-\fBwatchmalloc\fR(3MALLOC), 
-\fBattributes\fR(5), 
+For \fBmalloc()\fR, \fBcalloc()\fR, \fBfree()\fR, \fBrealloc()\fR, and
+\fBvalloc()\fR, see \fBstandards\fR(5).
+.SH SEE ALSO
+.sp
+.LP
+\fBexit\fR(2), \fBmmap\fR(2), \fBsbrk\fR(2), \fBbsdmalloc\fR(3MALLOC),
+\fBlibumem\fR(3LIB), \fBlongjmp\fR(3C), \fBmalloc\fR(3C),
+\fBmalloc\fR(3MALLOC), \fBmapmalloc\fR(3MALLOC), \fBpthread_exit\fR(3C),
+\fBthr_exit\fR(3C), \fBumem_cache_create\fR(3MALLOC),
+\fBumem_debug\fR(3MALLOC), \fBwatchmalloc\fR(3MALLOC), \fBattributes\fR(5),
 \fBstandards\fR(5)
 .sp
-
 .LP
-
-.sp
-
+\fISolaris Modular Debugger Guide\fR
 .SH WARNINGS
-
+.sp
 .LP
 Any of the following can cause undefined results:
-.sp
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-Passing a pointer returned from \fBumem_alloc()\fR or \fBumem_zalloc()\fR to \fBfree()\fR or \fBrealloc()\fR.
-.sp
-
+Passing a pointer returned from \fBumem_alloc()\fR or \fBumem_zalloc()\fR to
+\fBfree()\fR or \fBrealloc()\fR.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-Passing a pointer returned from \fBmalloc()\fR, \fBcalloc()\fR, \fBvalloc()\fR, \fBmemalign()\fR, or \fBrealloc()\fR to \fBumem_free()\fR.
-.sp
-
+Passing a pointer returned from \fBmalloc()\fR, \fBcalloc()\fR, \fBvalloc()\fR,
+\fBmemalign()\fR, or \fBrealloc()\fR to \fBumem_free()\fR.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-Writing past the end of a buffer allocated using \fBumem_alloc()\fR or \fBumem_zalloc()\fR
-.sp
-
+Writing past the end of a buffer allocated using \fBumem_alloc()\fR or
+\fBumem_zalloc()\fR
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-Performing \fBUMEM_NOFAIL\fR allocations from an 
-\fBatexit\fR(3C) handler.
-.sp
-
+Performing \fBUMEM_NOFAIL\fR allocations from an \fBatexit\fR(3C) handler.
 .RE
-
+.sp
 .LP
-If the \fBUMEM_NOFAIL\fR callback performs \fBUMEM_NOFAIL\fR allocations, infinite recursion can occur.
-.sp
-
+If the \fBUMEM_NOFAIL\fR callback performs \fBUMEM_NOFAIL\fR allocations,
+infinite recursion can occur.
 .SH NOTES
-
+.sp
 .LP
-The following list compares the features of the 
-\fBmalloc\fR(3C), 
-\fBbsdmalloc\fR(3MALLOC), 
-\fBmalloc\fR(3MALLOC), 
-\fBmtmalloc\fR(3MALLOC) , and the \fBlibumem\fR
-functions.
-.sp
-
-.sp
+The following list compares the features of the \fBmalloc\fR(3C),
+\fBbsdmalloc\fR(3MALLOC), \fBmalloc\fR(3MALLOC), \fBmtmalloc\fR(3MALLOC) , and
+the \fBlibumem\fR functions.
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The 
-\fBmalloc\fR(3C), 
-\fBbsdmalloc\fR(3MALLOC), and 
-\fBmalloc\fR(3MALLOC) functions have no support for concurrency. The \fBlibumem\fR and 
-\fBmtmalloc\fR(3MALLOC)
-functions support concurrent allocations.
-.sp
-
+The \fBmalloc\fR(3C), \fBbsdmalloc\fR(3MALLOC), and \fBmalloc\fR(3MALLOC)
+functions have no support for concurrency. The \fBlibumem\fR and
+\fBmtmalloc\fR(3MALLOC) functions support concurrent allocations.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The 
-\fBbsdmalloc\fR(3MALLOC) functions afford better performance but are space-inefficient.
-.sp
-
+The \fBbsdmalloc\fR(3MALLOC) functions afford better performance but are
+space-inefficient.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The 
-\fBmalloc\fR(3MALLOC) functions are space-efficient but have slower performance.
-.sp
-
+The \fBmalloc\fR(3MALLOC) functions are space-efficient but have slower
+performance.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The standard, fully SCD-compliant 
-\fBmalloc\fR(3C) functions are a trade-off between performance and space-efficiency.
-.sp
-
+The standard, fully SCD-compliant \fBmalloc\fR(3C) functions are a trade-off
+between performance and space-efficiency.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The 
-\fBmtmalloc\fR(3MALLOC) functions provide fast, concurrent \fBmalloc()\fR implementations that are not space-efficient.
-.sp
-
+The \fBmtmalloc\fR(3MALLOC) functions provide fast, concurrent \fBmalloc()\fR
+implementations that are not space-efficient.
 .RE
-
-.sp
 .RS +4
 .TP
 .ie t \(bu
 .el o
-The \fBlibumem\fR functions provide a fast, concurrent allocation implementation that in most cases is more space-efficient than 
+The \fBlibumem\fR functions provide a fast, concurrent allocation
+implementation that in most cases is more space-efficient than
 \fBmtmalloc\fR(3MALLOC).
-.sp
-
 .RE
-

umem_base.h

@@ -23,6 +23,9 @@
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ */
 
 #ifndef	_UMEM_BASE_H
 #define	_UMEM_BASE_H
@@ -76,6 +79,8 @@ extern volatile uint32_t umem_reaping;
 #define	UMEM_REAP_ADDING	0x00000001	/* umem_reap() is active */
 #define	UMEM_REAP_ACTIVE	0x00000002	/* update thread is reaping */
 
+extern uintptr_t umem_tmem_off;
+
 /*
  * umem.c: tunables
  */
@@ -98,6 +103,7 @@ extern size_t umem_lite_minsize;
 extern size_t umem_lite_maxalign;
 extern size_t umem_maxverify;
 extern size_t umem_minfirewall;
+extern size_t umem_ptc_size;
 
 extern uint32_t umem_flags;
 
@@ -140,6 +146,20 @@ extern int umem_create_update_thread(void);
 void umem_setup_envvars(int);
 void umem_process_envvars(void);
 
+/*
+ * umem_genasm.c: private interfaces
+ */
+extern const int umem_genasm_supported;
+extern int umem_genasm(int *, umem_cache_t **, int);
+
+/*
+ * malloc.c: traditional malloc/free interface for genasm
+ */
+extern void *umem_malloc(size_t);
+extern void umem_malloc_free(void *);
+extern void *_malloc(size_t);
+extern void _free(void *);
+
 #ifdef	__cplusplus
 }
 #endif

umem_debug.3

@@ -1,24 +1,9 @@
 '\" te
-.\" CDDL HEADER START
-.\"
-.\" The contents of this file are subject to the terms of the
-.\" Common Development and Distribution License (the "License").  
-.\" You may not use this file except in compliance with the License.
-.\"
-.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-.\" or http://www.opensolaris.org/os/licensing.
-.\" See the License for the specific language governing permissions
-.\" and limitations under the License.
-.\"
-.\" When distributing Covered Code, include this CDDL HEADER in each
-.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-.\" If applicable, add the following below this CDDL HEADER, with the
-.\" fields enclosed by brackets "[]" replaced with your own identifying
-.\" information: Portions Copyright [yyyy] [name of copyright owner]
-.\"
-.\" CDDL HEADER END
 .\" Copyright (c) 2002, Sun Microsystems, Inc. All Rights Reserved.
-.TH umem_debug 3MALLOC "26 July 2002" "SunOS 5.11" "Memory Allocation Library Functions"
+.\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License").  You may not use this file except in compliance with the License.
+.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing.  See the License for the specific language governing permissions and limitations under the License.
+.\" When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE.  If applicable, add the following below this CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner]
+.TH UMEM_DEBUG 3MALLOC "Jul 26, 2002"
 .SH NAME
 umem_debug \- debugging features of the umem library
 .SH SYNOPSIS
@@ -29,247 +14,198 @@ umem_debug \- debugging features of the umem library
 .fi
 
 .SH DESCRIPTION
-
-.LP
-The \fBlibumem\fR library provides debugging features that detect memory leaks, buffer overruns, multiple frees, use of uninitialized data, use of freed data, and many other common programming errors. The activation of the run-time debugging features is controlled by environment variables.
 .sp
-
 .LP
-When the library detects an error, it writes a description of the error to an internal buffer that is readable with the \fB::umem_status\fR 
-\fBmdb\fR(1) \fIdcmd\fR and then calls 
-\fBabort\fR(3C).
+The \fBlibumem\fR library provides debugging features that detect memory leaks,
+buffer overruns, multiple frees, use of uninitialized data, use of freed data,
+and many other common programming errors. The activation of the run-time
+debugging features is controlled by environment variables.
 .sp
-
+.LP
+When the library detects an error, it writes a description of the error to an
+internal buffer that is readable with the \fB::umem_status\fR \fBmdb\fR(1)
+\fIdcmd\fR and then calls \fBabort\fR(3C).
 .SH ENVIRONMENT VARIABLES
-
 .sp
 .ne 2
-.mk
 .na
-\fBUMEM_DEBUG\fR
+\fB\fBUMEM_DEBUG\fR\fR
 .ad
-.RS 14n
-.rt  
-This variable contains a list of comma-separated options.  Unrecognized options are ignored. Possible options include:
-.sp
-
+.RS 16n
+This variable contains a list of comma-separated options.  Unrecognized options
+are ignored. Possible options include:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBaudit\fR[=\fIframes\fR]\fR
 .ad
-.RS 18n
-.rt  
-This option enables the recording of auditing information, including thread ID, high-resolution time stamp, and stack trace for the last action (allocation or free) on every allocation.  If transaction logging
-(see UMEM_LOGGING) is enabled, this auditing information is also logged.
+.RS 20n
+This option enables the recording of auditing information, including thread ID,
+high-resolution time stamp, and stack trace for the last action (allocation or
+free) on every allocation.  If transaction logging (see \fBUMEM_LOGGING\fR) is
+enabled, this auditing information is also logged.
 .sp
-
+The \fIframes\fR parameter sets the number of stack frames recorded in the
+auditing structure. The upper bound for frames is implementation-defined. If a
+larger value is requested, the upper bound is used instead.
 .sp
-The \fIframes\fR parameter sets the number of stack frames recorded in the auditing structure. The upper bound for frames is implementation-defined. If a larger value is requested, the upper bound is used instead.
-.sp
-
-.sp
-If \fIframes\fR is not specified or is not an integer, the default value of 15 is used.
-.sp
-
+If \fIframes\fR is not specified or is not an integer, the default value of 15
+is used.
 .sp
 This option also enables the \fBguards\fR option.
-.sp
-
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcontents\fR[=\fIcount\fR]\fR
 .ad
-.RS 18n
-.rt  
-If auditing and contents logging (see UMEM_LOGGING) are enabled, the first \fIcount\fR bytes of each buffer are logged when they are freed.  If a buffer is shorter than \fIcount\fR bytes, it is logged in its entirety.
+.RS 20n
+If auditing and contents logging (see \fBUMEM_LOGGING\fR) are enabled, the
+first \fIcount\fR bytes of each buffer are logged when they are freed.  If a
+buffer is shorter than \fIcount\fR bytes, it is logged in its entirety.
 .sp
-
-.sp
-If \fIcount\fR is not specified or is not an integer, the default value of 256 is used.
-.sp
-
+If \fIcount\fR is not specified or is not an integer, the default value of 256
+is used.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBdefault\fR\fR
 .ad
-.RS 18n
-.rt  
+.RS 20n
 This option is equivalent to \fBaudit\fR,\fBcontents\fR,\fBguards\fR.
-.sp
-
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBguards\fR\fR
 .ad
-.RS 18n
-.rt  
-This option enables filling allocated and freed buffers with special patterns to help detect the use of uninitialized data and previously freed buffers. It also enables an 8-byte redzone after each buffer that contains \fB0xfeedfacefeedfaceULL\fR.
+.RS 20n
+This option enables filling allocated and freed buffers with special patterns
+to help detect the use of uninitialized data and previously freed buffers. It
+also enables an 8-byte redzone after each buffer that contains
+\fB0xfeedfacefeedfaceULL\fR.
 .sp
-
+When an object is freed, it is filled with \fB0xdeadbeef\fR.  When an object is
+allocated, the \fB0xdeadbeef\fR pattern is verified and replaced with
+\fB0xbaddcafe\fR.  The redzone is checked every time a buffer is allocated or
+freed.
 .sp
-When an object is freed, it is filled with \fB0xdeadbeef\fR.  When an object is allocated, the \fB0xdeadbeef\fR pattern is verified and replaced with \fB0xbaddcafe\fR.  The redzone is checked every time a buffer is allocated or freed.
-.sp
-
-.sp
-For caches with either constructors or destructors, or both, 
-\fBumem_cache_alloc\fR(3MALLOC) and 
-\fBumem_cache_free\fR(3MALLOC) apply the cache's constructor and destructor, respectively, instead of caching constructed objects. The presence of 
-\fBassert\fR(3C)s
-in the destructor verifying that the buffer is in the constructed state can be used to detect any objects returned in an improper state.  See 
-\fBumem_cache_create\fR(3MALLOC) for
-details.
-.sp
-
+For caches with either constructors or destructors, or both,
+\fBumem_cache_alloc\fR(3MALLOC) and \fBumem_cache_free\fR(3MALLOC) apply the
+cache's constructor and destructor, respectively, instead of caching
+constructed objects. The presence of \fBassert\fR(3C)s in the destructor
+verifying that the buffer is in the constructed state can be used to detect any
+objects returned in an improper state.  See \fBumem_cache_create\fR(3MALLOC)
+for details.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBverbose\fR\fR
 .ad
-.RS 18n
-.rt  
-The library writes error descriptions to standard error before aborting. These messages are not localized.
-.sp
-
+.RS 20n
+The library writes error descriptions to standard error before aborting. These
+messages are not localized.
 .RE
 
 .RE
 
 .sp
 .ne 2
-.mk
 .na
-\fBUMEM_LOGGING\fR
+\fB\fBUMEM_LOGGING\fR\fR
 .ad
-.RS 14n
-.rt  
-To be enabled, this variable should be set to a comma-separated list of in-memory logs. The logs available are:
-.sp
-
+.RS 16n
+To be enabled, this variable should be set to a comma-separated list of
+in-memory logs. The logs available are:
 .sp
 .ne 2
-.mk
 .na
 \fB\fBtransaction\fR[=\fIsize\fR]\fR
 .ad
-.RS 20n
-.rt  
-If the \fBaudit\fR debugging option is set (see \fBUMEM_DEBUG\fR), the audit structures from previous transactions are entered into this log.
-.sp
-
+.RS 22n
+If the \fBaudit\fR debugging option is set (see \fBUMEM_DEBUG\fR), the audit
+structures from previous transactions are entered into this log.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBcontents\fR[=\fIsize\fR]\fR
 .ad
-.RS 20n
-.rt  
-If the \fBaudit\fR debugging option is set, the contents of objects are recorded in this log as they are freed.
+.RS 22n
+If the \fBaudit\fR debugging option is set, the contents of objects are
+recorded in this log as they are freed.
 .sp
-
-.sp
-If the "contents" debugging option was not set, 256 bytes of each freed buffer are saved.
-.sp
-
+If the "contents" debugging option was not set, 256 bytes of each freed buffer
+are saved.
 .RE
 
 .sp
 .ne 2
-.mk
 .na
 \fB\fBfail\fR[=\fIsize\fR]\fR
 .ad
-.RS 20n
-.rt  
+.RS 22n
 Records are entered into this log for every failed allocation.
-.sp
-
 .RE
 
-For any of these options, if \fIsize\fR is not specified, the default value of 64k is used.  The \fIsize\fR parameter must be an integer that can be qualified with K, M, G, or T to specify kilobytes, megabytes, gigabytes, or terabytes, respectively.
+For any of these options, if \fIsize\fR is not specified, the default value of
+64k is used.  The \fIsize\fR parameter must be an integer that can be qualified
+with K, M, G, or T to specify kilobytes, megabytes, gigabytes, or terabytes,
+respectively.
 .sp
-
+Logs that are not listed or that have either a size of 0 or an invalid size are
+disabled.
 .sp
-Logs that are not listed or that have either a size of 0 or an invalid size are disabled.
-.sp
-
-.sp
-The log is disabled if during initialization the requested amount of storage cannot be allocated.
-.sp
-
+The log is disabled if during initialization the requested amount of storage
+cannot be allocated.
 .RE
 
 .SH ATTRIBUTES
-
-.LP
-See 
-\fBattributes\fR(5) for descriptions of the following attributes:
 .sp
-
 .LP
+See \fBattributes\fR(5) for descriptions of the following attributes:
+.sp
 
 .sp
 .TS
-tab() box;
-cw(2.75i) |cw(2.75i) 
-lw(2.75i) |lw(2.75i) 
-.
-ATTRIBUTE TYPEATTRIBUTE VALUE
+box;
+c | c
+l | l .
+ATTRIBUTE TYPE	ATTRIBUTE VALUE
 _
-Interface StabilityUnstable
+Interface Stability	Unstable
 _
-MT-LevelMT-Safe
+MT-Level	MT-Safe
 .TE
 
 .SH SEE ALSO
-
-.LP
-
-\fBmdb\fR(1), 
-\fBabort\fR(3C), 
-\fBsignal\fR(3C), 
-\fBumem_cache_create\fR(3MALLOC), 
-\fBattributes\fR(5)
 .sp
-
 .LP
-
+\fBmdb\fR(1), \fBabort\fR(3C), \fBsignal\fR(3C),
+\fBumem_cache_create\fR(3MALLOC), \fBattributes\fR(5)
 .sp
-
+.LP
+\fISolaris Modular Debugger Guide\fR
 .SH WARNINGS
-
-.LP
-When \fBlibumem\fR aborts the process using 
-\fBabort\fR(3C), any existing signal handler for \fBSIGABRT\fR is called. If the signal handler performs allocations, undefined
-behavior can result.
 .sp
-
+.LP
+When \fBlibumem\fR aborts the process using \fBabort\fR(3C), any existing
+signal handler for \fBSIGABRT\fR is called. If the signal handler performs
+allocations, undefined behavior can result.
 .SH NOTES
-
-.LP
-Some of the debugging features work only for allocations smaller than 16 kilobytes in size. Allocations larger than 16 kilobytes could have reduced support.
 .sp
-
 .LP
-Activating any of the library's debugging features could significantly increase the library's memory footprint and decrease its performance.
+Some of the debugging features work only for allocations smaller than 16
+kilobytes in size. Allocations larger than 16 kilobytes could have reduced
+support.
 .sp
-
+.LP
+Activating any of the library's debugging features could significantly increase
+the library's memory footprint and decrease its performance.

umem_impl.h

@@ -28,6 +28,11 @@
  * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
+/*
+ * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2015 by Delphix. All rights reserved.
+ */
+
 #ifndef _UMEM_IMPL_H
 #define	_UMEM_IMPL_H
 
@@ -45,7 +50,7 @@
 
 #include <sys/vmem.h>
 #ifdef HAVE_THREAD_H
-# include <thread.h>
+#include <thread.h>
 #else
 # include "sol_compat.h"
 #endif
@@ -78,6 +83,9 @@ extern "C" {
 
 #define	UMF_HASH	0x00000200	/* cache has hash table */
 #define	UMF_RANDOMIZE	0x00000400	/* randomize other umem_flags */
+#define	UMF_PTC		0x00000800	/* cache has per-thread caching */
+
+#define	UMF_CHECKNULL	0x00001000	/* heap exhaustion checking */
 
 #define	UMF_BUFTAG	(UMF_DEADBEEF | UMF_REDZONE)
 #define	UMF_TOUCH	(UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
@@ -418,6 +426,13 @@ extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
 extern int umem_add(caddr_t, size_t);
 #endif
 
+/*
+ * Private interface with libc for tcumem.
+ */
+extern uintptr_t _tmem_get_base(void);
+extern int _tmem_get_nentries(void);
+extern void _tmem_set_cleanup(void(*)(void *, int));
+
 #ifdef	__cplusplus
 }
 #endif

umem_test.c

@@ -4,10 +4,12 @@
 
 #include "umem.h"
 
+extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
+
 int main(int argc, char *argv[])
 {
   char *foo;
-  
+
   umem_startup(NULL, 0, 0, NULL, NULL);
 
   foo = umem_alloc(32, UMEM_DEFAULT);

umem_test2.c

@@ -3,6 +3,8 @@
 
 #include "umem.h"
 
+extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
+
 static const char *TESTSTRINGS[] = {
   "fred",
   "fredfredfred",
@@ -21,7 +23,7 @@ main (int argc, char *argv[])
   int i, j;
 
   memset(testcases, 0, sizeof(testcases));
-  
+
   umem_startup(NULL, 0, 0, NULL, NULL);
 
   for (i = 0; i < N_TESTSTRINGS; ++i)

vmem.c

@@ -22,8 +22,7 @@
 /*
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem.c	1.10	05/06/08 SMI" */

vmem_base.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_base.c	1.6	05/06/08 SMI" */

vmem_base.h

@@ -2,8 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, (the "License").
-   You may not use this file except in compliance with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -21,8 +21,7 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 #ifndef	_VMEM_BASE_H

vmem_sbrk.c

@@ -215,6 +215,9 @@ vmem_sbrk_tryfail(vmem_t *src, size_t size, int vmflags)
 static void *
 vmem_sbrk_alloc(vmem_t *src, size_t size, int vmflags)
 {
+	extern void *_sbrk_grow_aligned(size_t min_size, size_t low_align,
+	    size_t high_align, size_t *actual_size);
+
 	void *ret;
 	void *buf;
 	size_t buf_size;