First steps integrating per-thread caching work from the Illumos branch of libumem into my fork of portable-umem. Next up, a portable tmem (thread-local memory) implementation.

.gitignore

@@ -3,7 +3,6 @@
 *.la
 *.o
 *.libs
-INSTALL
 Makefile.in
 aclocal.m4
 autom4te.cache
@@ -11,10 +10,12 @@ config.guess
 config.h.in
 config.sub
 configure
+compile
 depcomp
 install-sh
 ltmain.sh
 missing
+m4
 .deps
 Doxyfile
 Makefile
@@ -23,6 +24,7 @@ config.log
 config.status
 libtool
 stamp-h1
+test-driver
 umem.spec
 umem_test
 umem_test1

INSTALL

@@ -0,0 +1 @@
+/usr/local/Cellar/automake/1.14.1/share/automake-1.14/INSTALL

Makefile.am

@@ -1,3 +1,5 @@
+ACLOCAL_AMFLAGS = -I m4
+
 lib_LTLIBRARIES = libumem.la libumem_malloc.la
 noinst_PROGRAMS = umem_test umem_test2 umem_test3
 
@@ -33,7 +35,10 @@ libumem_la_SOURCES =	init_lib.c \
 			sol_compat.h \
 			vmem.c \
 			sys/vmem.h \
-			sys/vmem_impl_user.h
+			sys/vmem_impl_user.h \
+			amd64/umem_genasm.c
+
+asm_libumem_sources =	amd64/asm_subr.s
 
 libumem_malloc_la_SOURCES = malloc.c
 libumem_malloc_la_LDFLAGS = -lpthread -R$(libdir) -lumem

README

@@ -22,3 +22,9 @@ Wez Furlong,
 Message Systems, Inc.
 wez (at) messagesystems (dot) com
 
+
+$ UMEM_OPTIONS=allocator=best ./umem_test
+Hello hello there
+
+UMEM_OPTIONS=allocator=best
+=best, =first, =next, or =instant

amd64/atomic.h

@@ -0,0 +1,104 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _ASM_ATOMIC_H
+#define	_ASM_ATOMIC_H
+
+#include <sys/ccompile.h>
+#include <sys/types.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if !defined(__lint) && defined(__GNUC__)
+
+#if defined(__amd64)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#ifdef notdef
+extern __GNU_INLINE uint64_t
+cas64(uint64_t *target, uint64_t cmp,
+	uint64_t newval)
+{
+	uint64_t retval;
+
+	__asm__ __volatile__(
+	    "movq %2, %%rax; lock; cmpxchgq %3, (%1)"
+	    : "=a" (retval)
+	    : "r" (target), "r" (cmp), "r" (newval));
+	return (retval);
+}
+#endif
+
+#elif defined(__i386)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#else
+#error	"port me"
+#endif
+
+#endif /* !__lint && __GNUC__ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _ASM_ATOMIC_H */

amd64/umem_genasm.c

@@ -0,0 +1,609 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you find the blocks of assembly that follow confusing and
+ * you're questioning why they exist, please go read section 8 of the umem.c big
+ * theory statement. Next familiarize yourself with the malloc and free
+ * implementations in libumem's malloc.c.
+ *
+ * What follows is the amd64 implementation of the thread caching automatic
+ * assembly generation. The amd64 calling conventions are documented in the
+ * 64-bit System V ABI. For our purposes what matters is that our first argument
+ * will come in rdi. Our functions have to preserve rbp, rbx, and r12->r15. We
+ * are free to do whatever we want with rax, rcx, rdx, rsi, rdi, and r8->r11.
+ *
+ * For both our implementation of malloc and free we only use the registers we
+ * don't have to preserve.
+ *
+ * Malloc register usage:
+ * 	o. rdi: Original size to malloc. This never changes and is preserved.
+ * 	o. rsi: Adjusted malloc size for malloc_data_tag(s).
+ * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. rdx: Pointer to the tmem_t array of roots
+ * 	o. r8:  Size of the cache
+ * 	o. r9:  Scratch register
+ *
+ * Free register usage:
+ *	o. rdi: Original buffer to free. This never changes and is preserved.
+ *	o. rax: The actual buffer, adjusted for the hidden malloc_data_t(s).
+ * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. rdx: Pointer to the tmem_t array of roots
+ * 	o. r8:  Size of the cache
+ * 	o. r9:  Scratch register
+ *
+ * Once we determine what cache we are using, we increment %rdx to the
+ * appropriate offset and set %r8 with the size of the cache. This means that
+ * when we break out to the normal buffer allocation point %rdx contains the
+ * head of the linked list and %r8 is the amount that we have to adjust the
+ * thread's cached amount by.
+ *
+ * Each block of assembly has psuedocode that describes its purpose.
+ */
+
+//include <atomic.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+int umem_genasm_supported = 1;
+uintptr_t umem_genasm_mptr;
+uintptr_t umem_genasm_msize;
+uintptr_t umem_genasm_fptr;
+uintptr_t umem_genasm_fsize;
+static uintptr_t umem_genasm_omptr;
+static uintptr_t umem_genasm_ofptr;
+
+#define	UMEM_GENASM_MAX64	(UINT32_MAX / sizeof (uintptr_t))
+#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
+#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
+#define	MULTINOP	 0x0000441f0f
+
+/*
+ * void *ptcmalloc(size_t orig_size);
+ *
+ * size_t size = orig_size + 8;
+ * if (size > UMEM_SECOND_ALIGN)
+ * 	size += 8;
+ *
+ * if (size < orig_size)
+ * 	goto tomalloc;		! This is overflow
+ *
+ * if (size > cache_max)
+ * 	goto tomalloc
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_MALINIT_JOUT	0x13
+#define	PTC_MALINIT_MCS	0x1a
+#define	PTC_MALINIT_JOV	0x20
+#define	PTC_MALINIT_SOFF	0x30
+static const uint8_t malinit[] =  {
+	0x48, 0x8d, 0x77, 0x08,		/* leaq 0x8(%rdi),%rsi */
+	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10, %rsi */
+	0x76, 0x04,			/* jbe +0x4 */
+	0x48, 0x8d, 0x77, 0x10,		/* leaq 0x10(%rdi),%rsi */
+	0x48, 0x39, 0xfe,		/* cmpq %rdi,%rsi */
+	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00,	/* jb +errout */
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
+	0x64, 0x48, 0x8b, 0x0c, 0x25,
+	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
+	0x48, 0x81, 0xc1,
+	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
+	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
+};
+
+/*
+ * void ptcfree(void *buf);
+ *
+ * if (buf == NULL)
+ * 	return;
+ *
+ * malloc_data_t *tag = buf;
+ * tag--;
+ * int size = tag->malloc_size;
+ * int tagval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
+ * if (tagval == MALLOC_SECOND_MAGIC) {
+ * 	tag--;
+ * } else if (tagval != MALLOC_MAGIC) {
+ * 	goto tofree;
+ * }
+ *
+ * if (size > cache_max)
+ * 	goto tofree;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_FRINI_JDONE	0x05
+#define	PTC_FRINI_JFREE	0x25
+#define	PTC_FRINI_MCS	0x30
+#define	PTC_FRINI_JOV	0x36
+#define	PTC_FRINI_SOFF	0x46
+static const uint8_t freeinit[] = {
+	0x48, 0x85, 0xff,		/* testq %rdi,%rdi */
+	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* jmp $JDONE (done) */
+	0x8b, 0x77, 0xf8,		/* movl -0x8(%rdi),%esi */
+	0x8b, 0x47, 0xfc,		/* movl -0x4(%rdi),%eax */
+	0x01, 0xf0,			/* addl %esi,%eax */
+	0x3d, 0x00, 0x70, 0xba, 0x16,	/* cmpl $MALLOC_2_MAGIC, %eax */
+	0x75, 0x06,			/* jne +0x6 (checkover) */
+	0x48, 0x8d, 0x47, 0xf0,		/* leaq -0x10(%rdi),%eax */
+	0xeb, 0x0f,			/* jmp +0xf (freebuf) */
+	0x3d, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl $MALLOC_MAGIC, %eax */
+	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jmp +JFREE (goto torfree) */
+	0x48, 0x8d, 0x47, 0xf8,		/* leaq -0x8(%rdi),%rax */
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
+	0x64, 0x48, 0x8b, 0x0c, 0x25,
+	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
+	0x48, 0x81, 0xc1,
+	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
+	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ * } else ...				! goto next cache
+ */
+#define	PTC_INICACHE_CMP	0x03
+#define	PTC_INICACHE_SIZE	0x0c
+#define	PTC_INICACHE_JMP	0x11
+static const uint8_t inicache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x0c,			/* ja +0xc (next cache) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0xe9, 0x00, 0x00, 0x00, 0x00,	/* jmp $JMP (allocbuf) */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else ...				! goto next cache
+ */
+#define	PTC_GENCACHE_CMP	0x03
+#define	PTC_GENCACHE_SIZE	0x0c
+#define	PTC_GENCACHE_NUM	0x13
+#define	PTC_GENCACHE_JMP	0x18
+static const uint8_t gencache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x14,			/* ja +0xc (next cache) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0x48, 0x81, 0xc2,
+	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp +$JMP (allocbuf ) */
+};
+
+/*
+ * else if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else {
+ *	goto tofunc; 			! goto tomalloc if ptcmalloc.
+ * }					! goto tofree if ptcfree.
+ */
+#define	PTC_FINCACHE_CMP	0x03
+#define	PTC_FINCACHE_JMP	0x08
+#define	PTC_FINCACHE_SIZE	0x0c
+#define	PTC_FINCACHE_NUM	0x13
+static const uint8_t fincache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x00,			/* ja +JMP (to real malloc) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0x48, 0x81, 0xc2,
+	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
+
+};
+
+/*
+ * if (*root == NULL)
+ * 	goto tomalloc;
+ *
+ * malloc_data_t *ret = *root;
+ * *root = *(void **)ret;
+ * t->tm_size += csize;
+ * ret->malloc_size = size;
+ *
+ * if (size > UMEM_SECOND_ALIGN) {
+ *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ *	ret += 2;
+ * } else {
+ *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ *	ret += 1;
+ * }
+ *
+ * return ((void *)ret);
+ * tomalloc:
+ * 	return (malloc(orig_size));
+ */
+#define	PTC_MALFINI_ALLABEL	0x00
+#define	PTC_MALFINI_JMLABEL	0x40
+#define	PTC_MALFINI_JMADDR	0x41
+static const uint8_t malfini[] = {
+	0x48, 0x8b, 0x02,		/* movl (%rdx),%rax */
+	0x48, 0x85, 0xc0,		/* testq %rax,%rax */
+	0x74, 0x38,			/* je +0x38 (errout) */
+	0x4c, 0x8b, 0x08,		/* movq (%rax),%r9 */
+	0x4c, 0x89, 0x0a,		/* movq %r9,(%rdx) */
+	0x4c, 0x29, 0x01,		/* subq %rsi,(%rcx) */
+	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10,%rsi */
+	0x76, 0x15,			/* jbe +0x15 */
+	0x41, 0xb9, 0x00, 0x70, 0xba, 0x16, /* movl $MALLOC_MAGIC_2, %r9d */
+	0x89, 0x70, 0x08,		/* movl %r9d,0x8(%rax) */
+	0x41, 0x29, 0xf1,		/* subl %esi, %r9d */
+	0x44, 0x89, 0x48, 0x0c,		/* movl %r9d, 0xc(%rax) */
+	0x48, 0x83, 0xc0, 0x10,		/* addq $0x10, %rax */
+	0xc3,				/* ret */
+	0x41, 0xb9, 0x00, 0xc0, 0x10, 0x3a,	/* movl %MALLOC_MAGIC, %r9d */
+	0x89, 0x30,			/* movl %esi,(%rax) */
+	0x41, 0x29, 0xf1,		/* subl %esi,%r9d */
+	0x44, 0x89, 0x48, 0x04,		/* movl %r9d,0x4(%rax) */
+	0x48, 0x83, 0xc0, 0x08,		/* addq $0x8,%rax */
+	0xc3,				/* ret */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $MALLOC */
+};
+
+/*
+ * if (t->tm_size + csize > umem_ptc_size)
+ * 	goto tofree;
+ *
+ * t->tm_size += csize
+ * *(void **)tag = *root;
+ * *root = tag;
+ * return;
+ * tofree:
+ * 	free(buf);
+ * 	return;
+ */
+#define	PTC_FRFINI_RBUFLABEL	0x00
+#define	PTC_FRFINI_CACHEMAX	0x09
+#define	PTC_FRFINI_DONELABEL	0x1b
+#define	PTC_FRFINI_JFLABEL	0x1c
+#define	PTC_FRFINI_JFADDR	0x1d
+static const uint8_t freefini[] = {
+	0x4c, 0x8b, 0x09,		/* movq (%rcx),%r9 */
+	0x4d, 0x01, 0xc1,		/* addq %r8, %r9 */
+	0x49, 0x81, 0xf9,
+	0x00, 0x00, 0x00, 0x00,		/* cmpl $THR_CACHE_MAX, %r9 */
+	0x77, 0x0d,			/* jae +0xd (torfree) */
+	0x4c, 0x01, 0x01,		/* addq %r8,(%rcx) */
+	0x4c, 0x8b, 0x0a,		/* movq (%rdx),%r9 */
+	0x4c, 0x89, 0x08,		/* movq %r9,(%rax) */
+	0x48, 0x89, 0x02,		/* movq %rax,(%rdx) */
+	0xc3,				/* ret */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp free */
+};
+
+/*
+ * Construct the initial part of malloc. off contains the offset from curthread
+ * to the root of the tmem structure. ep is the address of the label to error
+ * and jump to free. csize is the size of the largest umem_cache in ptcumem.
+ */
+static int
+genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
+{
+	uint32_t addr;
+
+	bcopy(malinit, bp, sizeof (malinit));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
+	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
+	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
+	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));
+
+	return (sizeof (malinit));
+}
+
+static int
+genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mcs)
+{
+	uint32_t addr;
+
+	bcopy(freeinit, bp, sizeof (freeinit));
+	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
+	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
+	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
+	bcopy(&mcs, bp + PTC_FRINI_MCS, sizeof (mcs));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
+	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
+	return (sizeof (freeinit));
+}
+
+
+/*
+ * Create the initial cache entry of the specified size. The value of ap tells
+ * us what the address of the label to try and allocate a buffer. This value is
+ * an offset from the current base to that value.
+ */
+static int
+genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+
+	bcopy(inicache, bp, sizeof (inicache));
+	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
+	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
+	ASSERT(addr != 0);
+	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));
+
+	return (sizeof (inicache));
+}
+
+static int
+genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+	uint32_t coff;
+
+	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
+	ASSERT(num != 0);
+	bcopy(gencache, bp, sizeof (gencache));
+	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
+	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
+	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));
+
+	return (sizeof (gencache));
+}
+
+static int
+genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
+{
+	uint8_t eap;
+	uint32_t coff;
+
+	ASSERT(ep <= 0xff && ep > 7);
+	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
+	bcopy(fincache, bp, sizeof (fincache));
+	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_FINCACHE_NUM, sizeof (coff));
+	eap = ep - PTC_FINCACHE_JMP - 1;
+	bcopy(&eap, bp + PTC_FINCACHE_JMP, sizeof (eap));
+
+	return (sizeof (fincache));
+}
+
+static int
+genasm_malfini(uint8_t *bp, uintptr_t mptr)
+{
+	uint32_t addr;
+
+	bcopy(malfini, bp, sizeof (malfini));
+	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
+	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));
+
+	return (sizeof (malfini));
+}
+
+static int
+genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
+{
+	uint32_t addr;
+
+	bcopy(freefini, bp, sizeof (freefini));
+	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
+	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
+	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));
+
+	return (sizeof (freefini));
+}
+
+/*
+ * The malloc inline assembly is constructed as follows:
+ *
+ * o Malloc prologue assembly
+ * o Generic first-cache check
+ * o n Generic cache checks (where n = _tmem_get_entries() - 2)
+ * o Generic last-cache check
+ * o Malloc epilogue assembly
+ *
+ * Generally there are at least three caches. When there is only one cache we
+ * only use the generic last-cache. In the case where there are two caches, we
+ * just leave out the middle ones.
+ */
+static int
+genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	int ii, off;
+	uint8_t *bp;
+	size_t total;
+	uint32_t allocoff, erroff;
+
+	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);
+
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
+	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;
+
+	bp = base;
+
+	off = genasm_malinit(bp, umem_tmem_off, erroff,
+	    umem_alloc_sizes[nents-1]);
+	bp += off;
+	allocoff -= off;
+	erroff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_malfini(bp, umem_genasm_omptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+static int
+genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	uint8_t *bp;
+	int ii, off;
+	size_t total;
+	uint32_t rbufoff, retoff, erroff;
+
+	/* Assume that nents has already been audited for us */
+	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
+	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
+	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);
+
+	bp = base;
+
+	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
+	    umem_alloc_sizes[nents - 1]);
+	bp += off;
+	erroff -= off;
+	rbufoff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
+		bp += off;
+		erroff -= off;
+		rbufoff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
+		bp += off;
+		rbufoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+/*ARGSUSED*/
+int
+umem_genasm(int *cp, umem_cache_t **caches, int nc)
+{
+	int nents, i;
+	uint8_t *mptr;
+	uint8_t *fptr;
+	uint32_t *ptr;
+	uint64_t v, *vptr;
+
+	mptr = (void *)((uintptr_t)&umem_genasm_mptr + 5);
+	fptr = (void *)((uintptr_t)&umem_genasm_fptr + 5);
+	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
+	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
+		return (1);
+
+	/*
+	 * The total number of caches that we can service is the minimum of:
+	 *  o the amount supported by libc
+	 *  o the total number of umem caches
+	 *  o we use a single byte addl, so its MAX_UINT32 / sizeof (uintptr_t).
+	 *    For 64-bit, this is MAX_UINT32 >> 3, a lot.
+	 */
+	nents = _tmem_get_nentries();
+
+	if (UMEM_GENASM_MAX64 < nents)
+		nents = UMEM_GENASM_MAX64;
+
+	if (nc < nents)
+		nents = nc;
+
+	/* Based on our constraints, this is not an error */
+	if (nents == 0 || umem_ptc_size == 0)
+		return (0);
+
+	/* Grab the original malloc and free locations */
+	ptr = (void *)(mptr - 4);
+	umem_genasm_omptr = *ptr + (uintptr_t)mptr;
+	ptr = (void *)(fptr - 4);
+	umem_genasm_ofptr = *ptr + (uintptr_t)fptr;
+
+	/* Take into account the jump */
+	if (genasm_malloc(mptr, umem_genasm_fsize - 5, nents, cp) != 0)
+		return (1);
+
+	if (genasm_free(fptr, umem_genasm_fsize - 5, nents, cp) != 0)
+		return (1);
+
+	/* nop out the jump with a multibyte jump */
+	vptr = (void *)&umem_genasm_mptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+	vptr = (void *)&umem_genasm_fptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+
+
+	for (i = 0; i < nents; i++)
+		caches[i]->cache_flags |= UMF_PTC;
+
+	return (0);
+}

atomic.h

@@ -0,0 +1,438 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_SYS_ATOMIC_H
+#define	_SYS_ATOMIC_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <sys/types.h>
+#include <sys/inttypes.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
+	(defined(__i386) || defined(__amd64))
+#include <asm/atomic.h>
+#endif
+
+#if defined(_KERNEL) || defined(__STDC__)
+/*
+ * Increment target.
+ */
+extern void atomic_inc_8(volatile uint8_t *);
+extern void atomic_inc_uchar(volatile uchar_t *);
+extern void atomic_inc_16(volatile uint16_t *);
+extern void atomic_inc_ushort(volatile ushort_t *);
+extern void atomic_inc_32(volatile uint32_t *);
+extern void atomic_inc_uint(volatile uint_t *);
+extern void atomic_inc_ulong(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_inc_64(volatile uint64_t *);
+#endif
+
+/*
+ * Decrement target
+ */
+extern void atomic_dec_8(volatile uint8_t *);
+extern void atomic_dec_uchar(volatile uchar_t *);
+extern void atomic_dec_16(volatile uint16_t *);
+extern void atomic_dec_ushort(volatile ushort_t *);
+extern void atomic_dec_32(volatile uint32_t *);
+extern void atomic_dec_uint(volatile uint_t *);
+extern void atomic_dec_ulong(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_dec_64(volatile uint64_t *);
+#endif
+
+/*
+ * Add delta to target
+ */
+extern void atomic_add_8(volatile uint8_t *, int8_t);
+extern void atomic_add_char(volatile uchar_t *, signed char);
+extern void atomic_add_16(volatile uint16_t *, int16_t);
+extern void atomic_add_short(volatile ushort_t *, short);
+extern void atomic_add_32(volatile uint32_t *, int32_t);
+extern void atomic_add_int(volatile uint_t *, int);
+extern void atomic_add_ptr(volatile void *, ssize_t);
+extern void atomic_add_long(volatile ulong_t *, long);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_add_64(volatile uint64_t *, int64_t);
+#endif
+
+/*
+ * logical OR bits with target
+ */
+extern void atomic_or_8(volatile uint8_t *, uint8_t);
+extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
+extern void atomic_or_16(volatile uint16_t *, uint16_t);
+extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
+extern void atomic_or_32(volatile uint32_t *, uint32_t);
+extern void atomic_or_uint(volatile uint_t *, uint_t);
+extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_or_64(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * logical AND bits with target
+ */
+extern void atomic_and_8(volatile uint8_t *, uint8_t);
+extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
+extern void atomic_and_16(volatile uint16_t *, uint16_t);
+extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
+extern void atomic_and_32(volatile uint32_t *, uint32_t);
+extern void atomic_and_uint(volatile uint_t *, uint_t);
+extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_and_64(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * As above, but return the new value.  Note that these _nv() variants are
+ * substantially more expensive on some platforms than the no-return-value
+ * versions above, so don't use them unless you really need to know the
+ * new value *atomically* (e.g. when decrementing a reference count and
+ * checking whether it went to zero).
+ */
+
+/*
+ * Increment target and return new value.
+ */
+extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
+extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
+extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
+extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
+extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
+extern uint_t atomic_inc_uint_nv(volatile uint_t *);
+extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
+#endif
+
+/*
+ * Decrement target and return new value.
+ */
+extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
+extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
+extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
+extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
+extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
+extern uint_t atomic_dec_uint_nv(volatile uint_t *);
+extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
+#endif
+
+/*
+ * Add delta to target
+ */
+extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
+extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
+extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
+extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
+extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
+extern uint_t atomic_add_int_nv(volatile uint_t *, int);
+extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
+extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
+#endif
+
+/*
+ * logical OR bits with target and return new value.
+ */
+extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
+extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
+extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
+extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
+extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
+extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
+extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * logical AND bits with target and return new value.
+ */
+extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
+extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
+extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
+extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
+extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
+extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
+extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * If *arg1 == arg2, set *arg1 = arg3; return old value
+ */
+extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
+extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
+extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
+extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
+extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
+extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
+extern void *atomic_cas_ptr(volatile void *, void *, void *);
+extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
+#endif
+
+/*
+ * Swap target and return old value
+ */
+extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
+extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
+extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
+extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
+extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
+extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
+extern void *atomic_swap_ptr(volatile void *, void *);
+extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * Perform an exclusive atomic bit set/clear on a target.
+ * Returns 0 if bit was sucessfully set/cleared, or -1
+ * if the bit was already set/cleared.
+ */
+extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
+extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
+
+/*
+ * Generic memory barrier used during lock entry, placed after the
+ * memory operation that acquires the lock to guarantee that the lock
+ * protects its data.  No stores from after the memory barrier will
+ * reach visibility, and no loads from after the barrier will be
+ * resolved, before the lock acquisition reaches global visibility.
+ */
+extern void membar_enter(void);
+
+/*
+ * Generic memory barrier used during lock exit, placed before the
+ * memory operation that releases the lock to guarantee that the lock
+ * protects its data.  All loads and stores issued before the barrier
+ * will be resolved before the subsequent lock update reaches visibility.
+ */
+extern void membar_exit(void);
+
+/*
+ * Arrange that all stores issued before this point in the code reach
+ * global visibility before any stores that follow; useful in producer
+ * modules that update a data item, then set a flag that it is available.
+ * The memory barrier guarantees that the available flag is not visible
+ * earlier than the updated data, i.e. it imposes store ordering.
+ */
+extern void membar_producer(void);
+
+/*
+ * Arrange that all loads issued before this point in the code are
+ * completed before any subsequent loads; useful in consumer modules
+ * that check to see if data is available and read the data.
+ * The memory barrier guarantees that the data is not sampled until
+ * after the available flag has been seen, i.e. it imposes load ordering.
+ */
+extern void membar_consumer(void);
+#endif
+
+#if !defined(_KERNEL) && !defined(__STDC__)
+extern void atomic_inc_8();
+extern void atomic_inc_uchar();
+extern void atomic_inc_16();
+extern void atomic_inc_ushort();
+extern void atomic_inc_32();
+extern void atomic_inc_uint();
+extern void atomic_inc_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_inc_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_dec_8();
+extern void atomic_dec_uchar();
+extern void atomic_dec_16();
+extern void atomic_dec_ushort();
+extern void atomic_dec_32();
+extern void atomic_dec_uint();
+extern void atomic_dec_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_dec_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_add_8();
+extern void atomic_add_char();
+extern void atomic_add_16();
+extern void atomic_add_short();
+extern void atomic_add_32();
+extern void atomic_add_int();
+extern void atomic_add_ptr();
+extern void atomic_add_long();
+#if defined(_INT64_TYPE)
+extern void atomic_add_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_or_8();
+extern void atomic_or_uchar();
+extern void atomic_or_16();
+extern void atomic_or_ushort();
+extern void atomic_or_32();
+extern void atomic_or_uint();
+extern void atomic_or_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_or_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_and_8();
+extern void atomic_and_uchar();
+extern void atomic_and_16();
+extern void atomic_and_ushort();
+extern void atomic_and_32();
+extern void atomic_and_uint();
+extern void atomic_and_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_and_64();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_inc_8_nv();
+extern uchar_t atomic_inc_uchar_nv();
+extern uint16_t atomic_inc_16_nv();
+extern ushort_t atomic_inc_ushort_nv();
+extern uint32_t atomic_inc_32_nv();
+extern uint_t atomic_inc_uint_nv();
+extern ulong_t atomic_inc_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_inc_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_dec_8_nv();
+extern uchar_t atomic_dec_uchar_nv();
+extern uint16_t atomic_dec_16_nv();
+extern ushort_t atomic_dec_ushort_nv();
+extern uint32_t atomic_dec_32_nv();
+extern uint_t atomic_dec_uint_nv();
+extern ulong_t atomic_dec_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_dec_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_add_8_nv();
+extern uchar_t atomic_add_char_nv();
+extern uint16_t atomic_add_16_nv();
+extern ushort_t atomic_add_short_nv();
+extern uint32_t atomic_add_32_nv();
+extern uint_t atomic_add_int_nv();
+extern void *atomic_add_ptr_nv();
+extern ulong_t atomic_add_long_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_add_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_or_8_nv();
+extern uchar_t atomic_or_uchar_nv();
+extern uint16_t atomic_or_16_nv();
+extern ushort_t atomic_or_ushort_nv();
+extern uint32_t atomic_or_32_nv();
+extern uint_t atomic_or_uint_nv();
+extern ulong_t atomic_or_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_or_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_and_8_nv();
+extern uchar_t atomic_and_uchar_nv();
+extern uint16_t atomic_and_16_nv();
+extern ushort_t atomic_and_ushort_nv();
+extern uint32_t atomic_and_32_nv();
+extern uint_t atomic_and_uint_nv();
+extern ulong_t atomic_and_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_and_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_cas_8();
+extern uchar_t atomic_cas_uchar();
+extern uint16_t atomic_cas_16();
+extern ushort_t atomic_cas_ushort();
+extern uint32_t atomic_cas_32();
+extern uint_t atomic_cas_uint();
+extern void *atomic_cas_ptr();
+extern ulong_t atomic_cas_ulong();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_cas_64();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_swap_8();
+extern uchar_t atomic_swap_uchar();
+extern uint16_t atomic_swap_16();
+extern ushort_t atomic_swap_ushort();
+extern uint32_t atomic_swap_32();
+extern uint_t atomic_swap_uint();
+extern void *atomic_swap_ptr();
+extern ulong_t atomic_swap_ulong();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_swap_64();
+#endif /* defined(_INT64_TYPE) */
+
+
+extern int atomic_set_long_excl();
+extern int atomic_clear_long_excl();
+
+extern void membar_enter();
+extern void membar_exit();
+extern void membar_producer();
+extern void membar_consumer();
+
+#endif
+
+#if defined(_KERNEL)
+
+#if defined(_LP64) || defined(_ILP32)
+#define	atomic_add_ip		atomic_add_long
+#define	atomic_add_ip_nv	atomic_add_long_nv
+#define	casip			atomic_cas_ulong
+#endif
+
+#if defined(__sparc)
+extern uint8_t ldstub(uint8_t *);
+#endif
+
+/*
+ * Legacy kernel interfaces; they will go away (eventually).
+ */
+extern uint8_t cas8(uint8_t *, uint8_t, uint8_t);
+extern uint32_t cas32(uint32_t *, uint32_t, uint32_t);
+extern uint64_t cas64(uint64_t *, uint64_t, uint64_t);
+extern ulong_t caslong(ulong_t *, ulong_t, ulong_t);
+extern void *casptr(void *, void *, void *);
+extern void atomic_and_long(ulong_t *, ulong_t);
+extern void atomic_or_long(ulong_t *, ulong_t);
+#if defined(__sparc)
+extern uint32_t swapl(uint32_t *, uint32_t);
+#endif
+
+#endif	/* _KERNEL */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_ATOMIC_H */

autogen.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-autoreconf -i -s
+autoreconf -fis

configure.ac

@@ -1,5 +1,6 @@
-AC_INIT([umem], [1.0.2], [], [umem])
+AC_INIT([umem], [1.0.3], [], [umem])
-AM_INIT_AUTOMAKE([dist-bzip2])
+AM_INIT_AUTOMAKE([dist-bzip2 subdir-objects])
+AC_CONFIG_MACRO_DIR([m4])
 
 AC_PROG_CC
 AM_PROG_AS

envvar.c

@@ -23,9 +23,9 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)envvar.c	1.5	05/06/08 SMI" */
@@ -181,7 +181,10 @@ static umem_env_item_t umem_options_items[] = {
 	},
 #endif
 #endif
-
+	{ "perthread_cache",	"Evolving",	ITEM_SIZE,
+		"Size (in bytes) of per-thread allocation cache",
+		NULL, 0, NULL, &umem_ptc_size
+	},
 	{ NULL, "-- end of UMEM_OPTIONS --",	ITEM_INVALID }
 };
 

getpcstack.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,12 +18,12 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- */
+ *
-/*
+ * Copyright 2006-2008 Message Systems, Inc.
- * Portions Copyright 2006-2008 Message Systems, Inc.
  */
 
 /* #pragma ident	"@(#)getpcstack.c	1.5	05/06/08 SMI" */

i386/asm_subr.s

@@ -0,0 +1,72 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <sys/asm_linkage.h>
+
+#if defined(lint)
+
+void *
+getfp(void)
+{
+	return (NULL);
+}
+
+#ifndef UMEM_STANDALONE
+void
+_breakpoint(void)
+{
+	return;
+}
+#endif
+
+#else	/* lint */
+
+#if defined(__amd64)
+
+	ENTRY(getfp)
+	movq	%rbp, %rax
+	ret
+	SET_SIZE(getfp)
+
+#else	/* __i386 */
+
+	ENTRY(getfp)
+	movl	%ebp, %eax
+	ret
+	SET_SIZE(getfp)
+
+#endif
+
+#ifndef UMEM_STANDALONE
+	ENTRY(_breakpoint)
+	int	$3
+	ret
+	SET_SIZE(_breakpoint)
+#endif
+
+#endif	/* lint */

i386/atomic.h

@@ -0,0 +1,104 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _ASM_ATOMIC_H
+#define	_ASM_ATOMIC_H
+
+#include <sys/ccompile.h>
+#include <sys/types.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if !defined(__lint) && defined(__GNUC__)
+
+#if defined(__amd64)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#ifdef notdef
+extern __GNU_INLINE uint64_t
+cas64(uint64_t *target, uint64_t cmp,
+	uint64_t newval)
+{
+	uint64_t retval;
+
+	__asm__ __volatile__(
+	    "movq %2, %%rax; lock; cmpxchgq %3, (%1)"
+	    : "=a" (retval)
+	    : "r" (target), "r" (cmp), "r" (newval));
+	return (retval);
+}
+#endif
+
+#elif defined(__i386)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#else
+#error	"port me"
+#endif
+
+#endif /* !__lint && __GNUC__ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _ASM_ATOMIC_H */

i386/umem_genasm.c

@@ -0,0 +1,603 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you find the blocks of assembly that follow confusing and
+ * you're questioning why they exist, please go read section 8 of the umem.c big
+ * theory statement. Next familiarize yourself with the malloc and free
+ * implementations in libumem's malloc.c.
+ *
+ * What follows is the i386 implementation of the thread caching automatic
+ * assembly generation. With i386 a function only has three registers its
+ * allowed to change without restoring them: eax, ecx, and edx. All others have
+ * to be preserved. Since the set of registers we have available is so small, we
+ * have to make use of esi, ebx, and edi and save their original values to the
+ * stack.
+ *
+ * Malloc register usage:
+ * 	o. esi: Size of the malloc (passed into us and modified)
+ * 	o. edi: Size of the cache
+ * 	o. eax: Buffer to return
+ * 	o. ebx: Scratch space and temporary values
+ * 	o. ecx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. edx: Pointer to the tmem_t array of roots
+ *
+ * Free register usage:
+ * 	o. esi: Size of the malloc (passed into us and modified)
+ * 	o. edi: Size of the cache
+ * 	o. eax: Buffer to free
+ * 	o. ebx: Scratch space and temporary values
+ * 	o. ecx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. edx: Pointer to the tmem_t array of roots
+ *
+ * Once we determine what cache we are using, we increment %edx to the
+ * appropriate offset and set %edi with the size of the cache. This means that
+ * when we break out to the normal buffer allocation point %edx contains the
+ * head of the linked list and %edi is the amount that we have to adjust the
+ * total amount cached by the thread.
+ *
+ * Each block of assembly has psuedocode that describes its purpose.
+ */
+
+#include <inttypes.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+#include <atomic.h>
+
+int umem_genasm_supported = 1;
+uintptr_t umem_genasm_mptr;
+size_t umem_genasm_msize;
+uintptr_t umem_genasm_fptr;
+size_t umem_genasm_fsize;
+static uintptr_t umem_genasm_omptr;
+static uintptr_t umem_genasm_ofptr;
+
+/*
+ * The maximum number of caches we can support. We use a single byte addl so
+ * this is 255 (UINT8_MAX) / sizeof (uintptr_t). In this case 63
+ */
+#define	UMEM_GENASM_MAX32	63
+
+#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
+#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
+#define	MULTINOP	 0x0000441f0f
+
+/*
+ * void *ptcmalloc(size_t orig_size);
+ *
+ * size_t size = orig_size + 8;
+ *
+ * if (size < orig_size)
+ * 	goto tomalloc;		! This is overflow
+ *
+ * if (size > cache_size)
+ * 	goto tomalloc;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_MALINIT_JOUT	0x0e
+#define	PTC_MALINIT_MCS	0x14
+#define	PTC_MALINIT_JOV	0x1a
+#define	PTC_MALINIT_SOFF	0x27
+static const uint8_t malinit[] = {
+	0x55,					/* pushl %ebp */
+	0x89, 0xe5,				/* movl %esp, %ebp */
+	0x57,					/* pushl %edi */
+	0x56,					/* pushl %esi */
+	0x53,					/* pushl %ebx */
+	0x8b, 0x75, 0x08,			/* movl 0x8(%ebp), %esi */
+	0x83, 0xc6, 0x08,			/* addl $0x8,%esi */
+	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00, 	/* jc +$JMP (errout) */
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +$JMP (errout) */
+	0x65, 0x8b, 0x0d, 0x00, 0x0, 0x00, 0x00, 	/* movl %gs:0x0,%ecx */
+	0x81, 0xc1, 0x00, 0x00,	0x00, 0x00, 	/* addl $OFF, %ecx */
+	0x8d, 0x51, 0x04			/* leal 0x4(%ecx), %edx */
+};
+
+/*
+ * void ptcfree(void *buf);
+ *
+ * if (buf == NULL)
+ * 	return;
+ *
+ * malloc_data_t *tag = buf;
+ * tag--;
+ * int size = tag->malloc_size;
+ * int tagtval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
+ *
+ * if (tagval != MALLOC_MAGIC)
+ * 	goto tofree;
+ *
+ * if (size > cache_max)
+ * 	goto tofree;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_FRINI_JDONE	0x0d
+#define	PTC_FRINI_JFREE	0x23
+#define	PTC_FRINI_MCS	0x29
+#define	PTC_FRINI_JOV	0x2f
+#define	PTC_FRINI_SOFF	0x3c
+static const uint8_t freeinit[] = {
+	0x55,					/* pushl %ebp */
+	0x89, 0xe5,				/* movl %esp, %ebp */
+	0x57,					/* pushl %edi */
+	0x56,					/* pushl %esi */
+	0x53,					/* pushl %ebx */
+	0x8b, 0x45, 0x08,			/* movl 0x8(%ebp), %eax */
+	0x85, 0xc0,				/* testl %eax, %eax */
+	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* je $JDONE (done) */
+	0x83, 0xe8, 0x08,			/* subl $0x8,%eax */
+	0x8b, 0x30,				/* movl (%eax),%esi */
+	0x8b, 0x50, 0x04,			/* movl 0x4(%eax),%edx */
+	0x01, 0xf2,				/* addl %esi,%edx */
+	0x81, 0xfa, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl MAGIC32, %edx */
+	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jne +JFREE (goto freebuf) */
+
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +$JMP (errout) */
+	0x65, 0x8b, 0x0d, 0x00, 0x0, 0x00, 0x00, /* movl %gs:0x0,%ecx */
+	0x81, 0xc1, 0x00, 0x00,	0x00, 0x00,	/* addl $0xOFF, %ecx */
+	0x8d, 0x51, 0x04			/* leal 0x4(%ecx),%edx */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ * } else ...				! goto next cache
+ */
+#define	PTC_INICACHE_CMP	0x02
+#define	PTC_INICACHE_SIZE 0x09
+#define	PTC_INICACHE_JMP	0x0e
+static const uint8_t inicache[] = {
+	0x81, 0xfe, 0xff, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x77, 0x0a,				/* ja +0xa */
+	0xbf, 0xff, 0x00, 0x00, 0x00, 		/* movl sizeof ($C0), %edi */
+	0xe9, 0x00, 0x00, 0x00, 0x00		/* jmp +$JMP (allocbuf) */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else ...				! goto next cache
+ */
+#define	PTC_GENCACHE_CMP	0x02
+#define	PTC_GENCACHE_NUM	0x0a
+#define	PTC_GENCACHE_SIZE 0x0c
+#define	PTC_GENCACHE_JMP	0x11
+static const uint8_t gencache[] = {
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($CACHE), %esi */
+	0x77, 0x0d,				/* ja +0xd (next cache) */
+	0x83, 0xc2, 0x00,			/* addl $4*$ii, %edx */
+	0xbf, 0x00, 0x00, 0x00, 0x00, 		/* movl sizeof ($CACHE), %edi */
+	0xe9, 0x00, 0x00, 0x00, 0x00 		/* jmp +$JMP (allocbuf) */
+};
+
+/*
+ * else if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else {
+ *	goto tofunc; 			! goto tomalloc if ptcmalloc.
+ * }					! goto tofree if ptcfree.
+ */
+#define	PTC_FINCACHE_CMP 0x02
+#define	PTC_FINCACHE_JMP	0x07
+#define	PTC_FINCACHE_NUM 0x0a
+#define	PTC_FINCACHE_SIZE 0x0c
+static const uint8_t fincache[] = {
+	0x81, 0xfe, 0xff, 0x00, 0x00, 0x00,	/* cmpl sizeof ($CLAST), %esi */
+	0x77, 0x00,				/* ja +$JMP (to errout) */
+	0x83, 0xc2, 0x00,			/* addl $4*($NCACHES-1), %edx */
+	0xbf, 0x00, 0x00, 0x00, 0x00, 		/* movl sizeof ($CLAST), %edi */
+};
+
+/*
+ * if (*root == NULL)
+ * 	goto tomalloc;
+ *
+ * malloc_data_t *ret = *root;
+ * *root = *(void **)ret;
+ * t->tm_size += csize;
+ * ret->malloc_size = size;
+ *
+ * ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ * ret++;
+ *
+ * return ((void *)ret);
+ * tomalloc:
+ * 	return (malloc(orig_size));
+ */
+#define	PTC_MALFINI_ALLABEL	0x00
+#define	PTC_MALFINI_JMLABEL	0x20
+#define	PTC_MALFINI_JMADDR	0x25
+static const uint8_t malfini[] = {
+	/* allocbuf: */
+	0x8b, 0x02,			/* movl (%edx), %eax */
+	0x85, 0xc0,			/* testl %eax, %eax */
+	0x74, 0x1a,			/* je +0x1a (errout) */
+	0x8b, 0x18,			/* movl (%eax), %esi */
+	0x89, 0x1a,			/* movl %esi, (%edx) */
+	0x29, 0x39,			/* subl %edi, (%ecx) */
+	0x89, 0x30,			/* movl %esi, ($eax) */
+	0xba, 0x00, 0xc0, 0x10, 0x3a,	/* movl $0x3a10c000,%edx */
+	0x29, 0xf2,			/* subl %esi, %edx */
+	0x89, 0x50, 0x04,		/* movl %edx, 0x4(%eax) */
+	0x83, 0xc0, 0x08,		/* addl %0x8, %eax */
+	0x5b,				/* popl %ebx */
+	0x5e,				/* popl %esi */
+	0x5f,				/* popl %edi */
+	0xc9,				/* leave */
+	0xc3,				/* ret */
+	/* errout: */
+	0x5b,				/* popl %ebx */
+	0x5e,				/* popl %esi */
+	0x5f,				/* popl %edi */
+	0xc9,				/* leave */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $malloc */
+};
+
+/*
+ * if (t->tm_size + csize > umem_ptc_size)
+ * 	goto tofree;
+ *
+ * t->tm_size += csize
+ * *(void **)tag = *root;
+ * *root = tag;
+ * return;
+ * tofree:
+ * 	free(buf);
+ * 	return;
+ */
+#define	PTC_FRFINI_RBUFLABEL	0x00
+#define	PTC_FRFINI_CACHEMAX	0x06
+#define	PTC_FRFINI_DONELABEL	0x14
+#define	PTC_FRFINI_JFLABEL	0x19
+#define	PTC_FRFINI_JFADDR	0x1e
+static const uint8_t freefini[] = {
+	/* freebuf: */
+	0x8b, 0x19,				/* movl (%ecx),%ebx */
+	0x01, 0xfb,				/* addl %edi,%ebx */
+	0x81, 0xfb, 0x00, 0x00, 0x00, 0x00, 	/* cmpl maxsize, %ebx */
+	0x73, 0x0d,				/* jae +0xd <tofree> */
+	0x01, 0x39,				/* addl %edi,(%ecx) */
+	0x8b, 0x3a,				/* movl (%edx),%edi */
+	0x89, 0x38,				/* movl %edi,(%eax) */
+	0x89, 0x02,				/* movl %eax,(%edx) */
+	/* done: */
+	0x5b,					/* popl %ebx */
+	0x5e,					/* popl %esi */
+	0x5f,					/* popl %edi */
+	0xc9,					/* leave */
+	0xc3,					/* ret */
+	/* realfree: */
+	0x5b,					/* popl %ebx */
+	0x5e,					/* popl %esi */
+	0x5f,					/* popl %edi */
+	0xc9,					/* leave */
+	0xe9, 0x00, 0x00, 0x00, 0x00		/* jmp free */
+};
+
+/*
+ * Construct the initial part of malloc. off contains the offset from curthread
+ * to the root of the tmem structure. ep is the address of the label to error
+ * and jump to free. csize is the size of the largest umem_cache in ptcumem.
+ */
+static int
+genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
+{
+	uint32_t addr;
+
+	bcopy(malinit, bp, sizeof (malinit));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
+	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
+	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
+	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));
+
+	return (sizeof (malinit));
+}
+
+static int
+genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mc)
+{
+	uint32_t addr;
+
+	bcopy(freeinit, bp, sizeof (freeinit));
+	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
+	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
+	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
+	bcopy(&mc, bp + PTC_FRINI_MCS, sizeof (mc));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
+	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
+	return (sizeof (freeinit));
+}
+
+/*
+ * Create the initial cache entry of the specified size. The value of ap tells
+ * us what the address of the label to try and allocate a buffer. This value is
+ * an offset from the current base to that value.
+ */
+static int
+genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+
+	bcopy(inicache, bp, sizeof (inicache));
+	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
+	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
+	ASSERT(addr != 0);
+	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));
+
+	return (sizeof (inicache));
+}
+
+static int
+genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+	uint8_t	coff;
+
+	ASSERT(256 / PTC_ROOT_SIZE > num);
+	ASSERT(num != 0);
+	bcopy(gencache, bp, sizeof (gencache));
+	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
+	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
+	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));
+
+	return (sizeof (gencache));
+}
+
+static int
+genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
+{
+	uint8_t addr;
+
+	ASSERT(ep <= 0xff && ep > 7);
+	ASSERT(256 / PTC_ROOT_SIZE > num);
+	bcopy(fincache, bp, sizeof (fincache));
+	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
+	addr = num * PTC_ROOT_SIZE;
+	bcopy(&addr, bp + PTC_FINCACHE_NUM, sizeof (addr));
+	addr = ep - PTC_FINCACHE_JMP - 1;
+	bcopy(&addr, bp + PTC_FINCACHE_JMP, sizeof (addr));
+
+	return (sizeof (fincache));
+}
+
+static int
+genasm_malfini(uint8_t *bp, uintptr_t mptr)
+{
+	uint32_t addr;
+
+	bcopy(malfini, bp, sizeof (malfini));
+	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
+	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));
+
+	return (sizeof (malfini));
+}
+
+static int
+genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
+{
+	uint32_t addr;
+
+	bcopy(freefini, bp, sizeof (freefini));
+	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
+	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
+	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));
+
+	return (sizeof (freefini));
+}
+
+/*
+ * The malloc inline assembly is constructed as follows:
+ *
+ * o Malloc prologue assembly
+ * o Generic first-cache check
+ * o n Generic cache checks (where n = _tmem_get_entries() - 2)
+ * o Generic last-cache check
+ * o Malloc epilogue assembly
+ *
+ * Generally there are at least three caches. When there is only one cache we
+ * only use the generic last-cache. In the case where there are two caches, we
+ * just leave out the middle ones.
+ */
+static int
+genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	int ii, off;
+	uint8_t *bp;
+	size_t total;
+	uint32_t allocoff, erroff;
+
+	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);
+
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
+	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;
+
+	bp = base;
+
+	off = genasm_malinit(bp, umem_tmem_off, erroff,
+	    umem_alloc_sizes[nents-1]);
+	bp += off;
+	allocoff -= off;
+	erroff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_malfini(bp, umem_genasm_omptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+static int
+genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	uint8_t *bp;
+	int ii, off;
+	size_t total;
+	uint32_t rbufoff, retoff, erroff;
+
+	/* Assume that nents has already been audited for us */
+	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
+	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
+	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);
+
+	bp = base;
+
+	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
+	    umem_alloc_sizes[nents - 1]);
+	bp += off;
+	erroff -= off;
+	rbufoff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
+		bp += off;
+		erroff -= off;
+		rbufoff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
+		bp += off;
+		rbufoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+int
+umem_genasm(int *alloc_sizes, umem_cache_t **caches, int ncaches)
+{
+	int nents, i;
+	uint8_t *mptr;
+	uint8_t *fptr;
+	uint32_t *ptr;
+	uint64_t v, *vptr;
+
+	mptr = (void *)((uintptr_t)&umem_genasm_mptr + 5);
+	fptr = (void *)((uintptr_t)&umem_genasm_fptr + 5);
+	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
+	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
+		return (1);
+
+	/*
+	 * The total number of caches that we can service is the minimum of:
+	 *  o the amount supported by libc
+	 *  o the total number of umem caches
+	 *  o we use a single byte addl, so its 255 / sizeof (uintptr_t). For
+	 *    32-bit, this is 63.
+	 */
+	nents = _tmem_get_nentries();
+
+	if (UMEM_GENASM_MAX32 < nents)
+		nents = UMEM_GENASM_MAX32;
+
+	if (ncaches < nents)
+		nents = ncaches;
+
+	/* Based on our constraints, this is not an error */
+	if (nents == 0 || umem_ptc_size == 0)
+		return (0);
+
+	/* Grab the original malloc and free locations */
+	ptr = (void *)(mptr - 4);
+	umem_genasm_omptr = *ptr + (uintptr_t)mptr;
+	ptr = (void *)(fptr - 4);
+	umem_genasm_ofptr = *ptr + (uintptr_t)fptr;
+
+	/* Take into account the jump */
+	if (genasm_malloc(mptr, umem_genasm_fsize - 5, nents,
+	    alloc_sizes) != 0)
+		return (1);
+
+	if (genasm_free(fptr, umem_genasm_fsize - 5, nents,
+	    alloc_sizes) != 0)
+		return (1);
+
+	/* nop out the jump with a multibyte jump */
+	vptr = (void *)&umem_genasm_mptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+	vptr = (void *)&umem_genasm_fptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+
+	for (i = 0; i < nents; i++)
+		caches[i]->cache_flags |= UMF_PTC;
+
+	return (0);
+}

init_lib.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)init_lib.c	1.2	05/06/08 SMI" */
@@ -149,7 +148,7 @@ umem_get_max_ncpus(void)
   return info.dwNumberOfProcessors;
 #else
   /* XXX: determine CPU count on other platforms */
-  return (1);
+  #error Cannot detremine CPU count on this platform, please submit a bug (and a patch) for this platform.
 #endif
 
 #endif /* linux */

init_stand.c

@@ -24,7 +24,7 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"@(#)init_stand.c	1.3	05/06/08 SMI"
+/* #pragma ident	"@(#)init_stand.c	1.3	05/06/08 SMI" */
 
 /*
  * Initialization routines for the standalone version of libumem.

linktest_stand.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -25,7 +24,7 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"@(#)linktest_stand.c	1.3	05/06/08 SMI"
+/* #pragma ident	"@(#)linktest_stand.c	1.3	05/06/08 SMI" */
 
 /*
  * This file is used to verify that the standalone's external dependencies
@@ -35,6 +34,7 @@
 void __umem_assert_failed(void) {}
 void _atomic_add_64(void) {}
 void _atomic_add_32_nv(void) {}
+void dladdr1(void) {}
 void bcopy(void) {}
 void bzero(void) {}
 void dladdr1(void) {}
@@ -43,6 +43,7 @@ void getenv(void) {}
 void gethrtime(void) {}
 void membar_producer(void) {}
 void memcpy(void) {}
+void _memcpy(void) {}
 void memset(void) {}
 void snprintf(void) {}
 void strchr(void) {}

malloc.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -25,7 +24,7 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"@(#)malloc.c	1.5	05/06/08 SMI"
+/* #pragma ident	"@(#)malloc.c	1.5	05/06/08 SMI" */
 
 #include "config.h"
 #include <unistd.h>

misc.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)misc.c	1.6	05/06/08 SMI" */

misc.h

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 #ifndef	_MISC_H
@@ -39,6 +38,7 @@
 #endif
 #ifdef HAVE_THREAD_H
 # include <thread.h>
+# include <pthread.h>
 #else
 # include "sol_compat.h"
 #endif

stand_mapfile

@@ -1,13 +1,11 @@
 #
-# Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
-# Use is subject to license terms.
 #
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
-# Common Development and Distribution License, Version 1.0 only
+# Common Development and Distribution License (the "License").
-# (the "License").  You may not use this file except in compliance
+# You may not use this file except in compliance with the License.
-# with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or http://www.opensolaris.org/os/licensing.

stub_stand.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -23,9 +22,11 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
  */
 
-#pragma ident	"@(#)stub_stand.c	1.3	05/06/08 SMI"
+/* #pragma ident	"@(#)stub_stand.c	1.3	05/06/08 SMI" */
 
 /*
  * Stubs for the standalone to reduce the dependence on external libraries
@@ -124,3 +125,29 @@ issetugid(void)
 {
 	return (1);
 }
+
+int
+_tmem_get_nentries(void)
+{
+	return (0);
+}
+
+uintptr_t
+_tmem_get_base(void)
+{
+	return (0);
+}
+
+/*ARGSUSED*/
+void
+_tmem_set_cleanup(void (*f)(int, void *))
+{
+}
+
+uint64_t
+atomic_swap_64(volatile uint64_t *t, uint64_t v)
+{
+	uint64_t old = *t;
+	*t = v;
+	return (old);
+}

sys/vmem_impl_user.h

@@ -23,7 +23,7 @@
  * Copyright 1999-2002 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 #ifndef _SYS_VMEM_IMPL_USER_H

tmem.c

@@ -0,0 +1,85 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ */
+
+#include "lint.h"
+#include "thr_uberdata.h"
+
+/*
+ * This file implements the private interface with libumem for per-thread
+ * caching umem (ptcumem). For the full details on how tcumem works and how
+ * these functions work, see section 8.4 of the big theory statement in
+ * lib/libumem/common/umem.c.
+ */
+static tmem_func_t tmem_cleanup = NULL;
+
+uintptr_t
+_tmem_get_base(void)
+{
+	return ((uintptr_t)&curthread->ul_tmem - (uintptr_t)curthread);
+}
+
+int
+_tmem_get_nentries(void)
+{
+	return (NTMEMBASE);
+}
+
+void
+_tmem_set_cleanup(tmem_func_t f)
+{
+	tmem_cleanup = f;
+}
+
+/*
+ * This is called by _thrp_exit() to clean up any per-thread allocations that
+ * are still hanging around and haven't been cleaned up.
+ */
+void
+tmem_exit(void)
+{
+	int ii;
+	void *buf, *next;
+	tumem_t *tp = &curthread->ul_tmem;
+
+
+	if (tp->tm_size == 0)
+		return;
+
+	/*
+	 * Since we have something stored here, we need to ensure we declared a
+	 * clean up handler. If we haven't that's broken and our single private
+	 * consumer should be shot.
+	 */
+	if (tmem_cleanup == NULL)
+		abort();
+	for (ii = 0; ii < NTMEMBASE; ii++) {
+		buf = tp->tm_roots[ii];
+		while (buf != NULL) {
+			next = *(void **)buf;
+			tmem_cleanup(buf, ii);
+			buf = next;
+		}
+	}
+}

umem.c

@@ -25,7 +25,7 @@
  * Portions Copyright 2012 Joyent, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident        "@(#)umem.c     1.11    05/06/08 SMI" */
@@ -78,7 +78,7 @@
  *
  * 1. Overview
  * -----------
- * umem is very close to kmem in implementation.  There are four major
+ * umem is very close to kmem in implementation.  There are seven major
  * areas of divergence:
  *
  *	* Initialization
@@ -91,6 +91,10 @@
  *
  *	* lock ordering
  *
+ *	* changing UMEM_MAXBUF
+ *
+ *	* Per-thread caching for malloc/free
+ *
  * 2. Initialization
  * -----------------
  * kmem is initialized early on in boot, and knows that no one will call
@@ -390,8 +394,6 @@
  *		lh_cpu[*].clh_lock
  *		lh_lock
  *
- * \endcode
- *
  * 7. Changing UMEM_MAXBUF
  * -----------------------
  *
@@ -402,6 +404,237 @@
  *
  * The second place to update, which is not required, is the umem_alloc_sizes.
  * These determine the default cache sizes that we're going to support.
+ *
+ * 8. Per-thread caching for malloc/free
+ * -------------------------------------
+ *
+ * "Time is an illusion. Lunchtime doubly so." -- Douglas Adams
+ *
+ * Time may be an illusion, but CPU cycles aren't.  While libumem is designed
+ * to be a highly scalable allocator, that scalability comes with a fixed cycle
+ * penalty even in the absence of contention: libumem must acquire (and release
+ * a per-CPU lock for each allocation.  When contention is low and malloc(3C)
+ * frequency is high, this overhead can dominate execution time.  To alleviate
+ * this, we allow for per-thread caching, a lock-free means of caching recent
+ * deallocations on a per-thread basis for use in satisfying subsequent calls
+ *
+ * In addition to improving performance, we also want to:
+ *	* Minimize fragmentation
+ *	* Not add additional memory overhead (no larger malloc tags)
+ *
+ * In the ulwp_t of each thread there is a private data structure called a
+ * umem_t that looks like:
+ *
+ * typedef struct {
+ * 	size_t	tm_size;
+ * 	void	*tm_roots[NTMEMBASE];  (Currently 16)
+ * } tmem_t;
+ *
+ * Each of the roots is treated as the head of a linked list. Each entry in the
+ * list can be thought of as a void ** which points to the next entry, until one
+ * of them points to NULL. If the head points to NULL, the list is empty.
+ *
+ * Each head corresponds to a umem_cache. Currently there is a linear mapping
+ * where the first root corresponds to the first cache, second root to the
+ * second cache, etc. This works because every allocation that malloc makes to
+ * umem_alloc that can be satisified by a umem_cache will actually return a
+ * number of bytes equal to the size of that cache. Because of this property and
+ * a one to one mapping between caches and roots we can guarantee that every
+ * entry in a given root's list will be able to satisfy the same requests as the
+ * corresponding cache.
+ *
+ * The maximum amount of memory that can be cached in each thread is determined
+ * by the perthread_cache UMEM_OPTION. It corresponds to the umem_ptc_size
+ * value. The default value for this is currently 1 MB. Once umem_init() has
+ * finished this cannot be directly tuned without directly modifying the
+ * instruction text. If, upon calling free(3C), the amount cached would exceed
+ * this maximum, we instead actually return the buffer to the umem_cache instead
+ * of holding onto it in the thread.
+ *
+ * When a thread calls malloc(3C) it first determines which umem_cache it
+ * would be serviced by. If the allocation is not covered by ptcumem it goes to
+ * the normal malloc instead.  Next, it checks if the tmem_root's list is empty
+ * or not. If it is empty, we instead go and allocate the memory from
+ * umem_alloc. If it is not empty, we remove the head of the list, set the
+ * appropriate malloc tags, and return that buffer.
+ *
+ * When a thread calls free(3C) it first looks at the malloc tag and if it is
+ * invalid or the allocation exceeds the largest cache in ptcumem and sends it
+ * off to the original free() to handle and clean up appropriately. Next, it
+ * checks if the allocation size is covered by one of the per-thread roots and
+ * if it isn't, it passes it off to the original free() to be released. Finally,
+ * before it inserts this buffer as the head, it checks if adding this buffer
+ * would put the thread over its maximum cache size. If it would, it frees the
+ * buffer back to the umem_cache. Otherwise it increments the threads total
+ * cached amount and makes the buffer the new head of the appropriate tm_root.
+ *
+ * When a thread exits, all of the buffers that it has in its per-thread cache
+ * will be passed to umem_free() and returned to the appropriate umem_cache.
+ *
+ * 8.1 Handling addition and removal of umem_caches
+ * ------------------------------------------------
+ *
+ * The set of umem_caches that are used to back calls to umem_alloc() and
+ * ultimately malloc() are determined at program execution time. The default set
+ * of caches is defined below in umem_alloc_sizes[]. Various umem_options exist
+ * that modify the set of caches: size_add, size_clear, and size_remove. Because
+ * the set of caches can only be determined once umem_init() has been called and
+ * we have the additional goals of minimizing additional fragmentation and
+ * metadata space overhead in the malloc tags, this forces our hand to go down a
+ * slightly different path: the one tread by fasttrap and trapstat.
+ *
+ * During umem_init we're going to dynamically construct a new version of
+ * malloc(3C) and free(3C) that utilizes the known cache sizes and then ensure
+ * that ptcmalloc and ptcfree replace malloc and free as entries in the plt. If
+ * ptcmalloc and ptcfree cannot handle a request, they simply jump to the
+ * original libumem implementations.
+ *
+ * After creating all of the umem_caches, but before making them visible,
+ * umem_cache_init checks that umem_genasm_supported is non-zero. This value is
+ * set by each architecture in $ARCH/umem_genasm.c to indicate whether or not
+ * they support this. If the value is zero, then this process is skipped.
+ * Similarly, if the cache size has been tuned to zero by UMEM_OPTIONS, then
+ * this is also skipped.
+ *
+ * In umem_genasm.c, each architecture's implementation implements a single
+ * function called umem_genasm() that is responsible for generating the
+ * appropriate versions of ptcmalloc() and ptcfree(), placing them in the
+ * appropriate memory location, and finally doing the switch from malloc() and
+ * free() to ptcmalloc() and ptcfree().  Once the change has been made, there is
+ * no way to switch back, short of restarting the program or modifying program
+ * text with mdb.
+ *
+ * 8.2 Modifying the Procedure Linkage Table (PLT)
+ * -----------------------------------------------
+ *
+ * The last piece of this puzzle is how we actually jam ptcmalloc() into the
+ * PLT.  The dyanmic linker has support for global and local audit libraries.
+ * For the full explanation of audit libraries consult the Linkers and Libraries
+ * guide or the linker source. A local auditer can attach to a single library
+ * and interpose on all of the relocations that come in from and leave to that
+ * same library. To facilitate our work, we have created a local audit library
+ * for libumem that is called libumem_trampoline and is located in
+ * lib/libumem_trampoline/.
+ *
+ * When any resolution is done to malloc(), the audit library allows us to
+ * replace the address with an address that it specifies. There are two 4k
+ * sections in the libumem_trampoline's bss which we use as the stomping grounds
+ * for ptcmalloc and ptcfree. When the audit library audits the malloc and free
+ * functions from libumem, it encodes their address and sets its buffers to
+ * contain a simple trampoline which consists of a jmp instruction and a four
+ * byte offset to the original malloc and free. libumem_trampoline's mapfile
+ * explicitly makes its bss rwx instead of rw to support this.
+ *
+ * When umem_genasm() is called, it uses a similar mechanism to get the address
+ * and size of the trampoline libraries malloc (mbuf) and free (fbuf) buffers.
+ * After validating that the size will be able to contain all of the
+ * instructions, it starts laying out ptcmalloc and ptcfree at mbuf[4] and
+ * fbuf[4]. Once both have been successfully generated, umem_genasm() stores a
+ * single five byte nop over the original jump.
+ *
+ * 8.3 umem_genasm()
+ * -----------------
+ *
+ * umem_genasm() is currently implemented for i386 and amd64. This section
+ * describes the theory behind the construction. For specific byte code to
+ * assembly instructions and niceish C and asm versions of ptcmalloc and
+ * ptcfree, see the individual umem_genasm.c files. The layout consists of the
+ * following sections:
+ *
+ *	o. function-specfic prologue
+ *	o. function-generic cache-selecting elements
+ *	o. function-specific epilogue
+ *
+ * There are three different generic cache elements that exist:
+ *
+ *	o. the last or only cache
+ *	o. the intermediary caches if more than two
+ *	o. the first one if more than one cache
+ *
+ * The malloc and free prologues and epilogues mimic the necessary portions of
+ * libumem's malloc and free. This includes things like checking for size
+ * overflow, setting and verifying the malloc tags.
+ *
+ * It is an important constraint that these functions do not make use of the
+ * call instruction. The only jmp outside of the individual functions is to the
+ * original libumem malloc and free respectively. Because doing things like
+ * setting errno or raising an internal umem error on improper malloc tags would
+ * require using calls into the PLT, whenever we encounter one of those cases we
+ * just jump to the original malloc and free functions reusing the same stack
+ * frame.
+ *
+ * Each of the above sections, the three caches, and the malloc and free
+ * prologue and epilogue are implemented as blocks of machine code with the
+ * corresponding assembly in comments. There are known offsets into each block
+ * that corresponds to locations of data and addresses that we only know at run
+ * time. These blocks are copied as necessary and the blanks filled in
+ * appropriately.
+ *
+ * As mentioned in section 8.2, the trampoline library uses specifically named
+ * variables to communicate the buffers and size to use. These variables are:
+ *
+ *	o. umem_genasm_mptr: The buffer for ptcmalloc
+ *	o. umem_genasm_msize: The size in bytes of the above buffer
+ *	o. umem_genasm_fptr: The buffer for ptcfree
+ *	o. umem_genasm_fsize: The size in bytes of the above buffer
+ *
+ * Finally, to enable the generated assembly we need to remove the previous jump
+ * to the actual malloc that exists at the start of these buffers. This is a
+ * five byte region. We could zero out the jump offset to be a jmp +0, but
+ * using nops can be faster. We specifically use a single five byte nop which is
+ * faster. The opcode for the five byte nop is 0x 0f 1f 44 00 00. On x86,
+ * remember integers are little endian, so it will be written the other way
+ * around.
+ *
+ * 8.4 Interface with libc.so
+ * --------------------------
+ *
+ * The tmem_t structure as described in the beginning of section 8, is part of a
+ * private interface with libc. There are three functions that exist to cover
+ * this. They are not documented in man pages or header files. They are in the
+ * SUNWprivate part of libc's makefile.
+ *
+ *	o. _tmem_get_base(void)
+ *
+ * 	Returns the offset from the ulwp_t (curthread) to the tmem_t structure.
+ * 	This is a constant for all threads and is effectively a way to to do
+ * 	::offsetof ulwp_t ul_tmem without having to know the specifics of the
+ * 	structure outside of libc.
+ *
+ *	o. _tmem_get_nentries(void)
+ *
+ *	Returns the number of roots that exist in the tmem_t. This is one part
+ *	of the cap on the number of umem_caches that we can back with tmem.
+ *
+ *	o. _tmem_set_cleanup(void (*)(void *, int))
+ *
+ *	This sets a clean up handler that gets called back when a thread exits.
+ *	There is one call per buffer, the void * is a pointer to the buffer on
+ *	the list, the int is the index into the roots array for this buffer.
+ *
+ * 8.5 Tuning and disabling per-thread caching
+ * -------------------------------------------
+ *
+ * There is only one tunable for per-thread caching:  the amount of memory each
+ * thread should be able to cache.  This is specified via the perthread_cache
+ * UMEM_OPTION option.  No attempt is made to to sanity check the specified
+ * value; the limit is simply the maximum value of a size_t.
+ *
+ * If the perthread_cache UMEM_OPTION is set to zero, nomagazines was requested,
+ * or UMEM_DEBUG has been turned on then we will never call into umem_genasm;
+ * however, the trampoline audit library and jump will still be in place.
+ *
+ * 8.6 Observing efficacy of per-thread caching
+ * --------------------------------------------
+ *
+ * To understand the efficacy of per-thread caching, use the ::umastat dcmd
+ * to see the percentage of capacity consumed on a per-thread basis, the
+ * degree to which each umem cache contributes to per-thread cache consumption,
+ * and the number of buffers in per-thread caches on a per-umem cache basis.
+ * If more detail is required, the specific buffers in a per-thread cache can
+ * be iterated over with the umem_ptc_* walkers. (These walkers allow an
+ * optional ulwp_t to be specified to iterate only over a particular thread's
+ * cache.)
  */
 
 #include "config.h"
@@ -524,8 +757,10 @@ size_t umem_lite_minsize = 0;   /* minimum buffer size for UMF_LITE */
 size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */
 size_t umem_maxverify;          /* maximum bytes to inspect in debug routines */
 size_t umem_minfirewall;        /* hardware-enforced redzone threshold */
+size_t umem_ptc_size = 1048576;	/* size of per-thread cache (in bytes) */
 
 uint_t umem_flags = 0;
+uintptr_t umem_tmem_off;
 
 mutex_t                 umem_init_lock = DEFAULTMUTEX;          /* locks initialization */
 cond_t                  umem_init_cv = DEFAULTCV;               /* initialization CV */
@@ -533,6 +768,8 @@ thread_t                umem_init_thr;          /* thread initializing */
 int                     umem_init_env_ready;    /* environ pre-initted */
 int                     umem_ready = UMEM_READY_STARTUP;
 
+int			umem_ptc_enabled;	/* per-thread caching enabled */
+
 static umem_nofail_callback_t *nofail_callback;
 static mutex_t          umem_nofail_exit_lock = DEFAULTMUTEX;
 static thread_t         umem_nofail_exit_thr;
@@ -2917,6 +3154,24 @@ umem_alloc_sizes_remove(size_t size)
         umem_alloc_sizes[i] = 0;
 }
 
+/*
+ * We've been called back from libc to indicate that thread is terminating and
+ * that it needs to release the per-thread memory that it has. We get to know
+ * which entry in the thread's tmem array the allocation came from. Currently
+ * this refers to first n umem_caches which makes this a pretty simple indexing
+ * job.
+ */
+static void
+umem_cache_tmem_cleanup(void *buf, int entry)
+{
+	size_t size;
+	umem_cache_t *cp;
+
+	size = umem_alloc_sizes[entry];
+	cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT];
+	_umem_cache_free(cp, buf);
+}
+
 static int
 umem_cache_init(void)
 {
@@ -3032,6 +3287,16 @@ umem_cache_init(void)
                 umem_alloc_caches[i] = cp;
         }
 
+	umem_tmem_off = _tmem_get_base();
+	_tmem_set_cleanup(umem_cache_tmem_cleanup);
+
+	if (umem_genasm_supported && !(umem_flags & UMF_DEBUG) &&
+	    !(umem_flags & UMF_NOMAGAZINE) &&
+	    umem_ptc_size > 0) {
+		umem_ptc_enabled = umem_genasm(umem_alloc_sizes,
+		    umem_alloc_caches, i) == 0 ? 1 : 0;
+	}
+
         /*
          * Initialization cannot fail at this point.  Make the caches
          * visible to umem_alloc() and friends.

umem_agent_support.c

@@ -23,7 +23,7 @@
  * Copyright 2002 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)umem_agent_support.c	1.2	05/06/08 SMI" */

umem_base.h

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -22,6 +21,8 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_UMEM_BASE_H
@@ -76,6 +77,8 @@ extern volatile uint32_t umem_reaping;
 #define	UMEM_REAP_ADDING	0x00000001	/* umem_reap() is active */
 #define	UMEM_REAP_ACTIVE	0x00000002	/* update thread is reaping */
 
+extern uintptr_t umem_tmem_off;
+
 /*
  * umem.c: tunables
  */
@@ -98,6 +101,7 @@ extern size_t umem_lite_minsize;
 extern size_t umem_lite_maxalign;
 extern size_t umem_maxverify;
 extern size_t umem_minfirewall;
+extern size_t umem_ptc_size;
 
 extern uint32_t umem_flags;
 
@@ -140,6 +144,12 @@ extern int umem_create_update_thread(void);
 void umem_setup_envvars(int);
 void umem_process_envvars(void);
 
+/*
+ * umem_genasm.c: private interfaces
+ */
+extern int umem_genasm_supported;
+extern int umem_genasm(int *, umem_cache_t **, int);
+
 #ifdef	__cplusplus
 }
 #endif

umem_fail.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
+ * Common Development and Distribution License (the "License").
- * (the "License").  You may not use this file except in compliance
+ * You may not use this file except in compliance with the License.
- * with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)umem_fail.c	1.4	05/06/08 SMI" */
@@ -166,5 +165,4 @@ __umem_assert_failed(const char *assertion, const char *file, int line)
 	umem_panic("Assertion failed: %s, file %s, line %d\n",
 	    assertion, file, line);
 	/*NOTREACHED*/
-	return (0);
 }

umem_impl.h

@@ -21,11 +21,11 @@
  */
 /*
  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
- *
- * Portions Copyright 2012 Joyent, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc.  All rights reserved.
+ *
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 #ifndef _UMEM_IMPL_H
@@ -78,6 +78,7 @@ extern "C" {
 
 #define	UMF_HASH	0x00000200	/* cache has hash table */
 #define	UMF_RANDOMIZE	0x00000400	/* randomize other umem_flags */
+#define	UMF_PTC		0x00000800	/* cache has per-thread caching */
 
 #define	UMF_BUFTAG	(UMF_DEADBEEF | UMF_REDZONE)
 #define	UMF_TOUCH	(UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
@@ -418,6 +419,13 @@ extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
 extern int umem_add(caddr_t, size_t);
 #endif
 
+/*
+ * Private interface with libc for tcumem.
+ */
+extern uintptr_t _tmem_get_base(void);
+extern int _tmem_get_nentries(void);
+extern void _tmem_set_cleanup(void(*)(void *, int));
+
 #ifdef	__cplusplus
 }
 #endif

umem_test.c

@@ -3,6 +3,8 @@
 #include <string.h>
 
 #include "umem.h"
+#define UMEM_STANDALONE
+#include "umem_impl.h"
 
 int main(int argc, char *argv[])
 {
@@ -20,4 +22,3 @@ int main(int argc, char *argv[])
 
   return EXIT_SUCCESS;
 }
-

umem_update_thread.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)umem_update_thread.c	1.2	05/06/08 SMI" */

umemdbg/mdb/common/libumem.c

@@ -0,0 +1,610 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ */
+
+#include "umem.h"
+#include <libproc.h>
+#include <mdb/mdb_modapi.h>
+
+#include "kgrep.h"
+#include "leaky.h"
+#include "misc.h"
+#include "proc_kludges.h"
+
+#include <umem_impl.h>
+#include <sys/vmem_impl_user.h>
+#include <thr_uberdata.h>
+#include <stdio.h>
+
+#include "umem_pagesize.h"
+
+typedef struct datafmt {
+	char	*hdr1;
+	char	*hdr2;
+	char	*dashes;
+	char	*fmt;
+} datafmt_t;
+
+static datafmt_t ptcfmt[] = {
+	{ "   ",	"tid",		"---",		"%3u "		},
+	{ " memory",	" cached",	"-------",	"%7lH "		},
+	{ "  %",	"cap",		"---",		"%3u "		},
+	{ "  %",	NULL,		"---",		"%3u "		},
+	{ NULL,		NULL,		NULL,		NULL		}
+};
+
+static datafmt_t umemfmt[] = {
+	{ "cache                    ", "name                     ",
+	"-------------------------", "%-25s "				},
+	{ "   buf",	"  size",	"------",	"%6u "		},
+	{ "    buf",	" in use",	"-------",	"%7u "		},
+	{ "    buf",	" in ptc",	"-------",	"%7s "		},
+	{ "    buf",	"  total",	"-------",	"%7u "		},
+	{ " memory",	" in use",	"-------",	"%7H "		},
+	{ "    alloc",	"  succeed",	"---------",	"%9u "		},
+	{ "alloc",	" fail",	"-----",	"%5llu"		},
+	{ NULL,		NULL,		NULL,		NULL		}
+};
+
+static datafmt_t vmemfmt[] = {
+	{ "vmem                     ", "name                     ",
+	"-------------------------", "%-*s "				},
+	{ "   memory",	"   in use",	"---------",	"%9H "		},
+	{ "    memory",	"     total",	"----------",	"%10H "		},
+	{ "   memory",	"   import",	"---------",	"%9H "		},
+	{ "    alloc",	"  succeed",	"---------",	"%9llu "	},
+	{ "alloc",	" fail",	"-----",	"%5llu "	},
+	{ NULL,		NULL,		NULL,		NULL		}
+};
+
+/*ARGSUSED*/
+static int
+umastat_cpu_avail(uintptr_t addr, const umem_cpu_cache_t *ccp, int *avail)
+{
+	if (ccp->cc_rounds > 0)
+		*avail += ccp->cc_rounds;
+	if (ccp->cc_prounds > 0)
+		*avail += ccp->cc_prounds;
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_cpu_alloc(uintptr_t addr, const umem_cpu_cache_t *ccp, int *alloc)
+{
+	*alloc += ccp->cc_alloc;
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_slab_avail(uintptr_t addr, const umem_slab_t *sp, int *avail)
+{
+	*avail += sp->slab_chunks - sp->slab_refcnt;
+
+	return (WALK_NEXT);
+}
+
+typedef struct umastat_vmem {
+	uintptr_t kv_addr;
+	struct umastat_vmem *kv_next;
+	int kv_meminuse;
+	int kv_alloc;
+	int kv_fail;
+} umastat_vmem_t;
+
+/*ARGSUSED*/
+static int
+umastat_cache_nptc(uintptr_t addr, const umem_cache_t *cp, int *nptc)
+{
+	if (!(cp->cache_flags & UMF_PTC))
+		return (WALK_NEXT);
+
+	(*nptc)++;
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_cache_hdr(uintptr_t addr, const umem_cache_t *cp, void *ignored)
+{
+	if (!(cp->cache_flags & UMF_PTC))
+		return (WALK_NEXT);
+
+	mdb_printf("%3d ", cp->cache_bufsize);
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_lwp_ptc(uintptr_t addr, void *buf, int *nbufs)
+{
+	(*nbufs)++;
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_lwp_cache(uintptr_t addr, const umem_cache_t *cp, ulwp_t *ulwp)
+{
+	char walk[60];
+	int nbufs = 0;
+
+	if (!(cp->cache_flags & UMF_PTC))
+		return (WALK_NEXT);
+
+	(void) snprintf(walk, sizeof (walk), "umem_ptc_%d", cp->cache_bufsize);
+
+	if (mdb_pwalk(walk, (mdb_walk_cb_t)umastat_lwp_ptc,
+	    &nbufs, (uintptr_t)ulwp->ul_self) == -1) {
+		mdb_warn("unable to walk '%s'", walk);
+		return (WALK_ERR);
+	}
+
+	mdb_printf("%3d ", ulwp->ul_tmem.tm_size ?
+	    (nbufs * cp->cache_bufsize * 100) / ulwp->ul_tmem.tm_size : 0);
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_lwp(uintptr_t addr, const ulwp_t *ulwp, void *ignored)
+{
+	size_t size;
+	datafmt_t *dfp = ptcfmt;
+
+	mdb_printf((dfp++)->fmt, ulwp->ul_lwpid);
+	mdb_printf((dfp++)->fmt, ulwp->ul_tmem.tm_size);
+
+	if (umem_readvar(&size, "umem_ptc_size") == -1) {
+		mdb_warn("unable to read 'umem_ptc_size'");
+		return (WALK_ERR);
+	}
+
+	mdb_printf((dfp++)->fmt, (ulwp->ul_tmem.tm_size * 100) / size);
+
+	if (mdb_walk("umem_cache",
+	    (mdb_walk_cb_t)umastat_lwp_cache, (void *)ulwp) == -1) {
+		mdb_warn("can't walk 'umem_cache'");
+		return (WALK_ERR);
+	}
+
+	mdb_printf("\n");
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_cache_ptc(uintptr_t addr, const void *ignored, int *nptc)
+{
+	(*nptc)++;
+	return (WALK_NEXT);
+}
+
+static int
+umastat_cache(uintptr_t addr, const umem_cache_t *cp, umastat_vmem_t **kvp)
+{
+	umastat_vmem_t *kv;
+	datafmt_t *dfp = umemfmt;
+	char buf[10];
+	int magsize;
+
+	int avail, alloc, total, nptc = 0;
+	size_t meminuse = (cp->cache_slab_create - cp->cache_slab_destroy) *
+	    cp->cache_slabsize;
+
+	mdb_walk_cb_t cpu_avail = (mdb_walk_cb_t)umastat_cpu_avail;
+	mdb_walk_cb_t cpu_alloc = (mdb_walk_cb_t)umastat_cpu_alloc;
+	mdb_walk_cb_t slab_avail = (mdb_walk_cb_t)umastat_slab_avail;
+
+	magsize = umem_get_magsize(cp);
+
+	alloc = cp->cache_slab_alloc + cp->cache_full.ml_alloc;
+	avail = cp->cache_full.ml_total * magsize;
+	total = cp->cache_buftotal;
+
+	(void) mdb_pwalk("umem_cpu_cache", cpu_alloc, &alloc, addr);
+	(void) mdb_pwalk("umem_cpu_cache", cpu_avail, &avail, addr);
+	(void) mdb_pwalk("umem_slab_partial", slab_avail, &avail, addr);
+
+	if (cp->cache_flags & UMF_PTC) {
+		char walk[60];
+
+		(void) snprintf(walk, sizeof (walk),
+		    "umem_ptc_%d", cp->cache_bufsize);
+
+		if (mdb_walk(walk,
+		    (mdb_walk_cb_t)umastat_cache_ptc, &nptc) == -1) {
+			mdb_warn("unable to walk '%s'", walk);
+			return (WALK_ERR);
+		}
+
+		(void) snprintf(buf, sizeof (buf), "%d", nptc);
+	}
+
+	for (kv = *kvp; kv != NULL; kv = kv->kv_next) {
+		if (kv->kv_addr == (uintptr_t)cp->cache_arena)
+			goto out;
+	}
+
+	kv = mdb_zalloc(sizeof (umastat_vmem_t), UM_SLEEP | UM_GC);
+	kv->kv_next = *kvp;
+	kv->kv_addr = (uintptr_t)cp->cache_arena;
+	*kvp = kv;
+out:
+	kv->kv_meminuse += meminuse;
+	kv->kv_alloc += alloc;
+	kv->kv_fail += cp->cache_alloc_fail;
+
+	mdb_printf((dfp++)->fmt, cp->cache_name);
+	mdb_printf((dfp++)->fmt, cp->cache_bufsize);
+	mdb_printf((dfp++)->fmt, total - avail);
+	mdb_printf((dfp++)->fmt, cp->cache_flags & UMF_PTC ? buf : "-");
+	mdb_printf((dfp++)->fmt, total);
+	mdb_printf((dfp++)->fmt, meminuse);
+	mdb_printf((dfp++)->fmt, alloc);
+	mdb_printf((dfp++)->fmt, cp->cache_alloc_fail);
+	mdb_printf("\n");
+
+	return (WALK_NEXT);
+}
+
+static int
+umastat_vmem_totals(uintptr_t addr, const vmem_t *v, umastat_vmem_t *kv)
+{
+	while (kv != NULL && kv->kv_addr != addr)
+		kv = kv->kv_next;
+
+	if (kv == NULL || kv->kv_alloc == 0)
+		return (WALK_NEXT);
+
+	mdb_printf("Total [%s]%*s %6s %7s %7s %7s %7H %9u %5u\n", v->vm_name,
+	    17 - strlen(v->vm_name), "", "", "", "", "",
+	    kv->kv_meminuse, kv->kv_alloc, kv->kv_fail);
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_vmem(uintptr_t addr, const vmem_t *v, void *ignored)
+{
+	datafmt_t *dfp = vmemfmt;
+	uintptr_t paddr;
+	vmem_t parent;
+	int ident = 0;
+
+	for (paddr = (uintptr_t)v->vm_source; paddr != NULL; ident += 4) {
+		if (mdb_vread(&parent, sizeof (parent), paddr) == -1) {
+			mdb_warn("couldn't trace %p's ancestry", addr);
+			ident = 0;
+			break;
+		}
+		paddr = (uintptr_t)parent.vm_source;
+	}
+
+	mdb_printf("%*s", ident, "");
+	mdb_printf((dfp++)->fmt, 25 - ident, v->vm_name);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_mem_inuse);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_mem_total);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_mem_import);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_alloc);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_fail);
+
+	mdb_printf("\n");
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+int
+umastat(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
+{
+	umastat_vmem_t *kv = NULL;
+	datafmt_t *dfp;
+	int nptc = 0, i;
+
+	if (argc != 0)
+		return (DCMD_USAGE);
+
+	/*
+	 * We need to determine if we have any caches that have per-thread
+	 * caching enabled.
+	 */
+	if (mdb_walk("umem_cache",
+	    (mdb_walk_cb_t)umastat_cache_nptc, &nptc) == -1) {
+		mdb_warn("can't walk 'umem_cache'");
+		return (DCMD_ERR);
+	}
+
+	if (nptc) {
+		for (dfp = ptcfmt; dfp->hdr2 != NULL; dfp++)
+			mdb_printf("%s ", dfp->hdr1);
+
+		for (i = 0; i < nptc; i++)
+			mdb_printf("%s ", dfp->hdr1);
+
+		mdb_printf("\n");
+
+		for (dfp = ptcfmt; dfp->hdr2 != NULL; dfp++)
+			mdb_printf("%s ", dfp->hdr2);
+
+		if (mdb_walk("umem_cache",
+		    (mdb_walk_cb_t)umastat_cache_hdr, NULL) == -1) {
+			mdb_warn("can't walk 'umem_cache'");
+			return (DCMD_ERR);
+		}
+
+		mdb_printf("\n");
+
+		for (dfp = ptcfmt; dfp->hdr2 != NULL; dfp++)
+			mdb_printf("%s ", dfp->dashes);
+
+		for (i = 0; i < nptc; i++)
+			mdb_printf("%s ", dfp->dashes);
+
+		mdb_printf("\n");
+
+		if (mdb_walk("ulwp", (mdb_walk_cb_t)umastat_lwp, NULL) == -1) {
+			mdb_warn("can't walk 'ulwp'");
+			return (DCMD_ERR);
+		}
+
+		mdb_printf("\n");
+	}
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->hdr1);
+	mdb_printf("\n");
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->hdr2);
+	mdb_printf("\n");
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->dashes);
+	mdb_printf("\n");
+
+	if (mdb_walk("umem_cache", (mdb_walk_cb_t)umastat_cache, &kv) == -1) {
+		mdb_warn("can't walk 'umem_cache'");
+		return (DCMD_ERR);
+	}
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->dashes);
+	mdb_printf("\n");
+
+	if (mdb_walk("vmem", (mdb_walk_cb_t)umastat_vmem_totals, kv) == -1) {
+		mdb_warn("can't walk 'vmem'");
+		return (DCMD_ERR);
+	}
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->dashes);
+	mdb_printf("\n");
+
+	mdb_printf("\n");
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->hdr1);
+	mdb_printf("\n");
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->hdr2);
+	mdb_printf("\n");
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->dashes);
+	mdb_printf("\n");
+
+	if (mdb_walk("vmem", (mdb_walk_cb_t)umastat_vmem, NULL) == -1) {
+		mdb_warn("can't walk 'vmem'");
+		return (DCMD_ERR);
+	}
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->dashes);
+	mdb_printf("\n");
+	return (DCMD_OK);
+}
+
+/*
+ * kmdb doesn't use libproc, and thus doesn't have any prmap_t's to walk.
+ * We have other ways to grep kmdb's address range.
+ */
+#ifndef _KMDB
+
+typedef struct ugrep_walk_data {
+	kgrep_cb_func *ug_cb;
+	void *ug_cbdata;
+} ugrep_walk_data_t;
+
+/*ARGSUSED*/
+int
+ugrep_mapping_cb(uintptr_t addr, const void *prm_arg, void *data)
+{
+	ugrep_walk_data_t *ug = data;
+	const prmap_t *prm = prm_arg;
+
+	return (ug->ug_cb(prm->pr_vaddr, prm->pr_vaddr + prm->pr_size,
+	    ug->ug_cbdata));
+}
+
+int
+kgrep_subr(kgrep_cb_func *cb, void *cbdata)
+{
+	ugrep_walk_data_t ug;
+
+	prockludge_add_walkers();
+
+	ug.ug_cb = cb;
+	ug.ug_cbdata = cbdata;
+
+	if (mdb_walk(KLUDGE_MAPWALK_NAME, ugrep_mapping_cb, &ug) == -1) {
+		mdb_warn("Unable to walk "KLUDGE_MAPWALK_NAME);
+		return (DCMD_ERR);
+	}
+
+	prockludge_remove_walkers();
+	return (DCMD_OK);
+}
+
+size_t
+kgrep_subr_pagesize(void)
+{
+	return (PAGESIZE);
+}
+
+#endif /* !_KMDB */
+
+static const mdb_dcmd_t dcmds[] = {
+
+	/* from libumem.c */
+	{ "umastat", NULL, "umem allocator stats", umastat },
+
+	/* from misc.c */
+	{ "umem_debug", NULL, "toggle umem dcmd/walk debugging", umem_debug},
+
+	/* from umem.c */
+	{ "umem_status", NULL, "Print umem status and message buffer",
+		umem_status },
+	{ "allocdby", ":", "given a thread, print its allocated buffers",
+		allocdby },
+	{ "bufctl", ":[-vh] [-a addr] [-c caller] [-e earliest] [-l latest] "
+		"[-t thd]", "print or filter a bufctl", bufctl, bufctl_help },
+	{ "bufctl_audit", ":", "print a bufctl_audit", bufctl_audit },
+	{ "freedby", ":", "given a thread, print its freed buffers", freedby },
+	{ "umalog", "[ fail | slab ]",
+	    "display umem transaction log and stack traces", umalog },
+	{ "umausers", "[-ef] [cache ...]", "display current medium and large "
+		"users of the umem allocator", umausers },
+	{ "umem_cache", "?", "print a umem cache", umem_cache },
+	{ "umem_log", "?", "dump umem transaction log", umem_log },
+	{ "umem_malloc_dist", "[-dg] [-b maxbins] [-B minbinsize]",
+		"report distribution of outstanding malloc()s",
+		umem_malloc_dist, umem_malloc_dist_help },
+	{ "umem_malloc_info", "?[-dg] [-b maxbins] [-B minbinsize]",
+		"report information about malloc()s by cache",
+		umem_malloc_info, umem_malloc_info_help },
+	{ "umem_verify", "?", "check integrity of umem-managed memory",
+		umem_verify },
+	{ "vmem", "?", "print a vmem_t", vmem },
+	{ "vmem_seg", ":[-sv] [-c caller] [-e earliest] [-l latest] "
+		"[-m minsize] [-M maxsize] [-t thread] [-T type]",
+		"print or filter a vmem_seg", vmem_seg, vmem_seg_help },
+
+#ifndef _KMDB
+	/* from ../genunix/kgrep.c + libumem.c */
+	{ "ugrep", KGREP_USAGE, "search user address space for a pointer",
+	    kgrep, kgrep_help },
+
+	/* from ../genunix/leaky.c + leaky_subr.c */
+	{ "findleaks", FINDLEAKS_USAGE, "search for potential memory leaks",
+	    findleaks, findleaks_help },
+#endif
+
+	{ NULL }
+};
+
+static const mdb_walker_t walkers[] = {
+
+	/* from umem.c */
+	{ "allocdby", "given a thread, walk its allocated bufctls",
+		allocdby_walk_init, allocdby_walk_step, allocdby_walk_fini },
+	{ "bufctl", "walk a umem cache's bufctls",
+		bufctl_walk_init, umem_walk_step, umem_walk_fini },
+	{ "bufctl_history", "walk the available history of a bufctl",
+		bufctl_history_walk_init, bufctl_history_walk_step,
+		bufctl_history_walk_fini },
+	{ "freectl", "walk a umem cache's free bufctls",
+		freectl_walk_init, umem_walk_step, umem_walk_fini },
+	{ "freedby", "given a thread, walk its freed bufctls",
+		freedby_walk_init, allocdby_walk_step, allocdby_walk_fini },
+	{ "freemem", "walk a umem cache's free memory",
+		freemem_walk_init, umem_walk_step, umem_walk_fini },
+	{ "umem", "walk a umem cache",
+		umem_walk_init, umem_walk_step, umem_walk_fini },
+	{ "umem_cpu", "walk the umem CPU structures",
+		umem_cpu_walk_init, umem_cpu_walk_step, umem_cpu_walk_fini },
+	{ "umem_cpu_cache", "given a umem cache, walk its per-CPU caches",
+		umem_cpu_cache_walk_init, umem_cpu_cache_walk_step, NULL },
+	{ "umem_hash", "given a umem cache, walk its allocated hash table",
+		umem_hash_walk_init, umem_hash_walk_step, umem_hash_walk_fini },
+	{ "umem_log", "walk the umem transaction log",
+		umem_log_walk_init, umem_log_walk_step, umem_log_walk_fini },
+	{ "umem_slab", "given a umem cache, walk its slabs",
+		umem_slab_walk_init, umem_slab_walk_step, NULL },
+	{ "umem_slab_partial",
+	    "given a umem cache, walk its partially allocated slabs (min 1)",
+		umem_slab_walk_partial_init, umem_slab_walk_step, NULL },
+	{ "vmem", "walk vmem structures in pre-fix, depth-first order",
+		vmem_walk_init, vmem_walk_step, vmem_walk_fini },
+	{ "vmem_alloc", "given a vmem_t, walk its allocated vmem_segs",
+		vmem_alloc_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+	{ "vmem_free", "given a vmem_t, walk its free vmem_segs",
+		vmem_free_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+	{ "vmem_postfix", "walk vmem structures in post-fix, depth-first order",
+		vmem_walk_init, vmem_postfix_walk_step, vmem_walk_fini },
+	{ "vmem_seg", "given a vmem_t, walk all of its vmem_segs",
+		vmem_seg_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+	{ "vmem_span", "given a vmem_t, walk its spanning vmem_segs",
+		vmem_span_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+
+#ifndef _KMDB
+	/* from ../genunix/leaky.c + leaky_subr.c */
+	{ "leak", "given a leak ctl, walk other leaks w/ that stacktrace",
+		leaky_walk_init, leaky_walk_step, leaky_walk_fini },
+	{ "leakbuf", "given a leak ctl, walk addr of leaks w/ that stacktrace",
+		leaky_walk_init, leaky_buf_walk_step, leaky_walk_fini },
+#endif
+
+	{ NULL }
+};
+
+static const mdb_modinfo_t modinfo = {MDB_API_VERSION, dcmds, walkers};
+
+const mdb_modinfo_t *
+_mdb_init(void)
+{
+	if (umem_init() != 0)
+		return (NULL);
+
+	return (&modinfo);
+}
+
+void
+_mdb_fini(void)
+{
+#ifndef _KMDB
+	leaky_cleanup(1);
+#endif
+}

umemdbg/mdb/common/misc.c

@@ -0,0 +1,110 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include "misc.h"
+
+#define	UMEM_OBJNAME "libumem.so"
+
+int umem_debug_level = 0;
+int umem_is_standalone = 0;
+
+/*ARGSUSED*/
+int
+umem_debug(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
+{
+	umem_debug_level ^= 1;
+
+	mdb_printf("umem: debugging is now %s\n",
+	    umem_debug_level ? "on" : "off");
+
+	return (DCMD_OK);
+}
+
+/*
+ * To further confuse the issue, this dmod can run against either
+ * libumem.so.1 *or* the libstandumem.so linked into kmdb(1M).  To figure
+ * out which one we are working against, we look up "umem_alloc" in both
+ * libumem.so and the executable.
+ *
+ * A further wrinkle is that libumem.so may not yet be loaded into the
+ * process' address space.  That can lead to either the lookup failing, or
+ * being unable to read from the data segment.  We treat either case as
+ * an error.
+ */
+int
+umem_set_standalone(void)
+{
+	GElf_Sym sym;
+	int ready;
+
+	if (mdb_lookup_by_obj(UMEM_OBJNAME, "umem_alloc", &sym) == 0)
+		umem_is_standalone = 0;
+	else if (mdb_lookup_by_obj(MDB_OBJ_EXEC, "umem_alloc", &sym) == 0)
+		umem_is_standalone = 1;
+	else
+		return (-1);
+
+	/*
+	 * now that we know where things should be, make sure we can actually
+	 * read things out.
+	 */
+	if (umem_readvar(&ready, "umem_ready") == -1)
+		return (-1);
+	return (0);
+}
+
+ssize_t
+umem_lookup_by_name(const char *name, GElf_Sym *sym)
+{
+	return (mdb_lookup_by_obj((umem_is_standalone ? MDB_OBJ_EXEC :
+	    UMEM_OBJNAME), name, sym));
+}
+
+/* This is like mdb_readvar, only for libumem.so's symbols */
+ssize_t
+umem_readvar(void *buf, const char *name)
+{
+	GElf_Sym sym;
+
+	if (umem_lookup_by_name(name, &sym))
+		return (-1);
+
+	if (mdb_vread(buf, sym.st_size, (uintptr_t)sym.st_value)
+	    == sym.st_size)
+		return ((ssize_t)sym.st_size);
+
+	return (-1);
+}
+
+int
+is_umem_sym(const char *sym, const char *prefix)
+{
+	char *tick_p = strrchr(sym, '`');
+
+	return (strncmp(sym, "libumem", 7) == 0 && tick_p != NULL &&
+	    strncmp(tick_p + 1, prefix, strlen(prefix)) == 0);
+}

umemdbg/mdb/common/misc.h

@@ -0,0 +1,67 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MDBMOD_MISC_H
+#define	_MDBMOD_MISC_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <mdb/mdb_modapi.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	offsetof(s, m)	((size_t)(&(((s *)0)->m)))
+
+extern int umem_debug(uintptr_t, uint_t, int, const mdb_arg_t *);
+
+extern int umem_set_standalone(void);
+extern ssize_t umem_lookup_by_name(const char *, GElf_Sym *);
+extern ssize_t umem_readvar(void *, const char *);
+
+/*
+ * Returns non-zero if sym matches libumem*`prefix*
+ */
+int is_umem_sym(const char *, const char *);
+
+#define	dprintf(x) if (umem_debug_level) { \
+	mdb_printf("umem debug: ");  \
+	/*CSTYLED*/\
+	mdb_printf x ;\
+}
+
+#define	dprintf_cont(x) if (umem_debug_level) { \
+	/*CSTYLED*/\
+	mdb_printf x ;\
+}
+
+extern int umem_debug_level;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _MDBMOD_MISC_H */

umemdbg/mdb/common/proc_kludges.c

@@ -0,0 +1,165 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2000-2002 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <mdb/mdb_modapi.h>
+#include <sys/machelf.h>
+
+#include <libproc.h>
+#include <stdio.h>
+
+#include "proc_kludges.h"
+
+typedef struct prockuldge_mappings {
+	struct ps_prochandle *pkm_Pr;
+
+	uint_t pkm_idx;
+
+	uint_t pkm_count;
+	uint_t pkm_max;
+
+	prmap_t *pkm_mappings;
+
+	uint_t pkm_old_max;
+	prmap_t *pkm_old_mappings;
+} prockludge_mappings_t;
+
+/* ARGSUSED */
+static int
+prockludge_mappings_iter(prockludge_mappings_t *pkm, const prmap_t *pmp,
+    const char *object_name)
+{
+	if (pkm->pkm_count >= pkm->pkm_max) {
+		int s = pkm->pkm_max ? pkm->pkm_max * 2 : 16;
+
+		pkm->pkm_old_max = pkm->pkm_max;
+		pkm->pkm_old_mappings = pkm->pkm_mappings;
+		pkm->pkm_max = s;
+		pkm->pkm_mappings = mdb_alloc(sizeof (prmap_t) * s, UM_SLEEP);
+
+		bcopy(pkm->pkm_old_mappings, pkm->pkm_mappings,
+		    sizeof (prmap_t) * pkm->pkm_old_max);
+
+		mdb_free(pkm->pkm_old_mappings,
+		    sizeof (prmap_t) * pkm->pkm_old_max);
+
+		pkm->pkm_old_mappings = NULL;
+		pkm->pkm_old_max = 0;
+	}
+	bcopy(pmp, &pkm->pkm_mappings[pkm->pkm_count++], sizeof (prmap_t));
+
+	return (0);
+}
+
+int
+prockludge_mappings_walk_init(mdb_walk_state_t *mws)
+{
+	struct ps_prochandle *Pr;
+	int rc;
+
+	prockludge_mappings_t *pkm;
+
+	if (mdb_get_xdata("pshandle", &Pr, sizeof (Pr)) == -1) {
+		mdb_warn("couldn't read pshandle xdata");
+		return (WALK_ERR);
+	}
+
+	pkm = mdb_zalloc(sizeof (prockludge_mappings_t), UM_SLEEP);
+	pkm->pkm_Pr = Pr;
+	mws->walk_data = pkm;
+
+	rc = Pmapping_iter(Pr, (proc_map_f *)prockludge_mappings_iter, pkm);
+	if (rc != 0) {
+		mdb_warn("Pmapping_iter failed");
+		/* clean up */
+		prockludge_mappings_walk_fini(mws);
+		return (WALK_ERR);
+	}
+	return (WALK_NEXT);
+}
+
+int
+prockludge_mappings_walk_step(mdb_walk_state_t *wsp)
+{
+	prockludge_mappings_t *pkm = wsp->walk_data;
+	int status;
+
+	if (pkm->pkm_idx >= pkm->pkm_count)
+		return (WALK_DONE);
+
+	status = wsp->walk_callback(0, &pkm->pkm_mappings[pkm->pkm_idx++],
+	    wsp->walk_cbdata);
+
+	return (status);
+}
+
+void
+prockludge_mappings_walk_fini(mdb_walk_state_t *wsp)
+{
+	prockludge_mappings_t *pkm = wsp->walk_data;
+	if (pkm != NULL) {
+		if (pkm->pkm_old_mappings != NULL) {
+			mdb_free(pkm->pkm_old_mappings,
+			    sizeof (prmap_t) * pkm->pkm_old_max);
+		}
+		if (pkm->pkm_mappings &&
+		    pkm->pkm_mappings != pkm->pkm_old_mappings) {
+			mdb_free(pkm->pkm_mappings,
+			    sizeof (prmap_t) * pkm->pkm_max);
+		}
+		mdb_free(pkm, sizeof (prockludge_mappings_t));
+	}
+}
+
+static int add_count = 0;
+
+void
+prockludge_add_walkers(void)
+{
+	mdb_walker_t w;
+
+	if (add_count++ == 0) {
+		w.walk_name = KLUDGE_MAPWALK_NAME;
+		w.walk_descr = "kludge: walk the process' prmap_ts";
+		w.walk_init = prockludge_mappings_walk_init;
+		w.walk_step = prockludge_mappings_walk_step;
+		w.walk_fini = prockludge_mappings_walk_fini;
+		w.walk_init_arg = NULL;
+
+		if (mdb_add_walker(&w) == -1) {
+			mdb_warn("unable to add walker "KLUDGE_MAPWALK_NAME);
+		}
+	}
+}
+
+void
+prockludge_remove_walkers(void)
+{
+	if (--add_count == 0) {
+		mdb_remove_walker(KLUDGE_MAPWALK_NAME);
+	}
+}

umemdbg/mdb/common/proc_kludges.h

@@ -0,0 +1,49 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2000-2002 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_PROC_KLUDGES_H
+#define	_PROC_KLUDGES_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#define	KLUDGE_MAPWALK_NAME "__prockludge_mappings"
+
+extern int prockludge_mappings_walk_init(mdb_walk_state_t *);
+extern int prockludge_mappings_walk_step(mdb_walk_state_t *);
+extern void prockludge_mappings_walk_fini(mdb_walk_state_t *);
+
+extern void prockludge_add_walkers(void);
+extern void prockludge_remove_walkers(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _PROC_KLUDGES_H */

umemdbg/mdb/common/umem.h

@@ -0,0 +1,140 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MDBMOD_UMEM_H
+#define	_MDBMOD_UMEM_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <mdb/mdb_modapi.h>
+#include <umem_impl.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int umem_ready;
+extern uint32_t umem_stack_depth;
+
+extern int umem_cache_walk_init(mdb_walk_state_t *);
+extern int umem_cache_walk_step(mdb_walk_state_t *);
+extern void umem_cache_walk_fini(mdb_walk_state_t *);
+
+extern int umem_cpu_walk_init(mdb_walk_state_t *);
+extern int umem_cpu_walk_step(mdb_walk_state_t *);
+extern void umem_cpu_walk_fini(mdb_walk_state_t *);
+
+extern int umem_cpu_cache_walk_init(mdb_walk_state_t *);
+extern int umem_cpu_cache_walk_step(mdb_walk_state_t *);
+
+extern int umem_slab_walk_init(mdb_walk_state_t *);
+extern int umem_slab_walk_partial_init(mdb_walk_state_t *);
+extern int umem_slab_walk_step(mdb_walk_state_t *);
+
+extern int umem_hash_walk_init(mdb_walk_state_t *wsp);
+extern int umem_hash_walk_step(mdb_walk_state_t *wsp);
+extern void umem_hash_walk_fini(mdb_walk_state_t *wsp);
+
+extern int umem_walk_init(mdb_walk_state_t *);
+extern int bufctl_walk_init(mdb_walk_state_t *);
+extern int freemem_walk_init(mdb_walk_state_t *);
+extern int freectl_walk_init(mdb_walk_state_t *);
+
+extern int umem_walk_step(mdb_walk_state_t *);
+extern void umem_walk_fini(mdb_walk_state_t *);
+
+extern int bufctl_history_walk_init(mdb_walk_state_t *);
+extern int bufctl_history_walk_step(mdb_walk_state_t *);
+extern void bufctl_history_walk_fini(mdb_walk_state_t *);
+
+extern int allocdby_walk_init(mdb_walk_state_t *);
+extern int allocdby_walk_step(mdb_walk_state_t *);
+extern void allocdby_walk_fini(mdb_walk_state_t *);
+
+extern int freedby_walk_init(mdb_walk_state_t *);
+extern int freedby_walk_step(mdb_walk_state_t *);
+extern void freedby_walk_fini(mdb_walk_state_t *);
+
+extern int umem_log_walk_init(mdb_walk_state_t *);
+extern int umem_log_walk_step(mdb_walk_state_t *);
+extern void umem_log_walk_fini(mdb_walk_state_t *);
+
+extern int allocdby_walk_init(mdb_walk_state_t *);
+extern int allocdby_walk_step(mdb_walk_state_t *);
+extern void allocdby_walk_fini(mdb_walk_state_t *);
+
+extern int freedby_walk_init(mdb_walk_state_t *);
+extern int freedby_walk_step(mdb_walk_state_t *);
+extern void freedby_walk_fini(mdb_walk_state_t *);
+
+extern int vmem_walk_init(mdb_walk_state_t *);
+extern int vmem_walk_step(mdb_walk_state_t *);
+extern void vmem_walk_fini(mdb_walk_state_t *);
+
+extern int vmem_postfix_walk_step(mdb_walk_state_t *);
+
+extern int vmem_seg_walk_init(mdb_walk_state_t *);
+extern int vmem_seg_walk_step(mdb_walk_state_t *);
+extern void vmem_seg_walk_fini(mdb_walk_state_t *);
+
+extern int vmem_span_walk_init(mdb_walk_state_t *);
+extern int vmem_alloc_walk_init(mdb_walk_state_t *);
+extern int vmem_free_walk_init(mdb_walk_state_t *);
+
+extern int allocdby(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int bufctl(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int bufctl_audit(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int freedby(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umalog(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umausers(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_cache(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_log(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_malloc_dist(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_malloc_info(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_status(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_verify(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_verify_alloc(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_verify_free(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int vmem(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int vmem_seg(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int whatis(uintptr_t, uint_t, int, const mdb_arg_t *);
+
+extern void bufctl_help(void);
+extern void umem_malloc_dist_help(void);
+extern void umem_malloc_info_help(void);
+extern void vmem_seg_help(void);
+
+/*
+ * utility functions for the rest of libumem
+ */
+extern int umem_init(void);
+extern int umem_get_magsize(const umem_cache_t *);
+extern size_t umem_estimate_allocated(uintptr_t, const umem_cache_t *);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _MDBMOD_UMEM_H */

umemdbg/mdb/common/umem_pagesize.h

@@ -0,0 +1,44 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2002 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_UMEM_PAGESIZE_H
+#define	_UMEM_PAGESIZE_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+extern size_t umem_pagesize;
+#undef PAGESIZE
+#define	PAGESIZE (umem_pagesize)
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _UMEM_PAGESIZE_H */

vmem.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem.c	1.10	05/06/08 SMI" */

vmem_base.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_base.c	1.6	05/06/08 SMI" */

vmem_base.h

@@ -2,8 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, (the "License").
+ * Common Development and Distribution License (the "License").
-   You may not use this file except in compliance with the License.
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -22,7 +22,7 @@
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 #ifndef	_VMEM_BASE_H

vmem_mmap.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_mmap.c	1.2	05/06/08 SMI" */

vmem_sbrk.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_sbrk.c	1.4	05/06/08 SMI" */