First steps integrating per-thread caching work from the Illumos branch of libumem into my fork of portable-umem. Next up, a portable tmem (thread-local memory) implementation.

.gitignore

@@ -3,7 +3,6 @@
 *.la
 *.o
 *.libs
-INSTALL
 Makefile.in
 aclocal.m4
 autom4te.cache
@@ -11,10 +10,12 @@ config.guess
 config.h.in
 config.sub
 configure
+compile
 depcomp
 install-sh
 ltmain.sh
 missing
+m4
 .deps
 Doxyfile
 Makefile
@@ -23,6 +24,7 @@ config.log
 config.status
 libtool
 stamp-h1
+test-driver
 umem.spec
 umem_test
 umem_test1

INSTALL

@@ -0,0 +1 @@
+/usr/local/Cellar/automake/1.14.1/share/automake-1.14/INSTALL

Makefile.am

@@ -1,3 +1,5 @@
+ACLOCAL_AMFLAGS = -I m4
+
 lib_LTLIBRARIES = libumem.la libumem_malloc.la
 noinst_PROGRAMS = umem_test umem_test2 umem_test3
 
@@ -33,10 +35,13 @@ libumem_la_SOURCES =	init_lib.c \
 			sol_compat.h \
 			vmem.c \
 			sys/vmem.h \
-			sys/vmem_impl_user.h
+			sys/vmem_impl_user.h \
+			amd64/umem_genasm.c
+
+asm_libumem_sources =	amd64/asm_subr.s
 
 libumem_malloc_la_SOURCES = malloc.c
-libumem_malloc_la_LDFLAGS = -lpthread -R$(libdir) -lumem 
+libumem_malloc_la_LDFLAGS = -lpthread -R$(libdir) -lumem
 
 man3_MANS = umem_alloc.3 umem_cache_create.3 umem_debug.3
 EXTRA_DIST = COPYRIGHT OPENSOLARIS.LICENSE umem.spec Doxyfile umem_test4 \

README

@@ -17,8 +17,14 @@ Linux and Windows versions of our product as we have found it to be stable.
 We will continue to update this project as and when we make improvements, and
 welcome third-party patches that improve the usability for everyone.
 
- 
+
 Wez Furlong,
 Message Systems, Inc.
 wez (at) messagesystems (dot) com
 
+
+$ UMEM_OPTIONS=allocator=best ./umem_test
+Hello hello there
+
+UMEM_OPTIONS=allocator=best
+=best, =first, =next, or =instant

amd64/atomic.h

@@ -0,0 +1,104 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _ASM_ATOMIC_H
+#define	_ASM_ATOMIC_H
+
+#include <sys/ccompile.h>
+#include <sys/types.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if !defined(__lint) && defined(__GNUC__)
+
+#if defined(__amd64)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#ifdef notdef
+extern __GNU_INLINE uint64_t
+cas64(uint64_t *target, uint64_t cmp,
+	uint64_t newval)
+{
+	uint64_t retval;
+
+	__asm__ __volatile__(
+	    "movq %2, %%rax; lock; cmpxchgq %3, (%1)"
+	    : "=a" (retval)
+	    : "r" (target), "r" (cmp), "r" (newval));
+	return (retval);
+}
+#endif
+
+#elif defined(__i386)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#else
+#error	"port me"
+#endif
+
+#endif /* !__lint && __GNUC__ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _ASM_ATOMIC_H */

amd64/umem_genasm.c

@@ -0,0 +1,609 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you find the blocks of assembly that follow confusing and
+ * you're questioning why they exist, please go read section 8 of the umem.c big
+ * theory statement. Next familiarize yourself with the malloc and free
+ * implementations in libumem's malloc.c.
+ *
+ * What follows is the amd64 implementation of the thread caching automatic
+ * assembly generation. The amd64 calling conventions are documented in the
+ * 64-bit System V ABI. For our purposes what matters is that our first argument
+ * will come in rdi. Our functions have to preserve rbp, rbx, and r12->r15. We
+ * are free to do whatever we want with rax, rcx, rdx, rsi, rdi, and r8->r11.
+ *
+ * For both our implementation of malloc and free we only use the registers we
+ * don't have to preserve.
+ *
+ * Malloc register usage:
+ * 	o. rdi: Original size to malloc. This never changes and is preserved.
+ * 	o. rsi: Adjusted malloc size for malloc_data_tag(s).
+ * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. rdx: Pointer to the tmem_t array of roots
+ * 	o. r8:  Size of the cache
+ * 	o. r9:  Scratch register
+ *
+ * Free register usage:
+ *	o. rdi: Original buffer to free. This never changes and is preserved.
+ *	o. rax: The actual buffer, adjusted for the hidden malloc_data_t(s).
+ * 	o. rcx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. rdx: Pointer to the tmem_t array of roots
+ * 	o. r8:  Size of the cache
+ * 	o. r9:  Scratch register
+ *
+ * Once we determine what cache we are using, we increment %rdx to the
+ * appropriate offset and set %r8 with the size of the cache. This means that
+ * when we break out to the normal buffer allocation point %rdx contains the
+ * head of the linked list and %r8 is the amount that we have to adjust the
+ * thread's cached amount by.
+ *
+ * Each block of assembly has psuedocode that describes its purpose.
+ */
+
+//include <atomic.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+int umem_genasm_supported = 1;
+uintptr_t umem_genasm_mptr;
+uintptr_t umem_genasm_msize;
+uintptr_t umem_genasm_fptr;
+uintptr_t umem_genasm_fsize;
+static uintptr_t umem_genasm_omptr;
+static uintptr_t umem_genasm_ofptr;
+
+#define	UMEM_GENASM_MAX64	(UINT32_MAX / sizeof (uintptr_t))
+#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
+#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
+#define	MULTINOP	 0x0000441f0f
+
+/*
+ * void *ptcmalloc(size_t orig_size);
+ *
+ * size_t size = orig_size + 8;
+ * if (size > UMEM_SECOND_ALIGN)
+ * 	size += 8;
+ *
+ * if (size < orig_size)
+ * 	goto tomalloc;		! This is overflow
+ *
+ * if (size > cache_max)
+ * 	goto tomalloc
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_MALINIT_JOUT	0x13
+#define	PTC_MALINIT_MCS	0x1a
+#define	PTC_MALINIT_JOV	0x20
+#define	PTC_MALINIT_SOFF	0x30
+static const uint8_t malinit[] =  {
+	0x48, 0x8d, 0x77, 0x08,		/* leaq 0x8(%rdi),%rsi */
+	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10, %rsi */
+	0x76, 0x04,			/* jbe +0x4 */
+	0x48, 0x8d, 0x77, 0x10,		/* leaq 0x10(%rdi),%rsi */
+	0x48, 0x39, 0xfe,		/* cmpq %rdi,%rsi */
+	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00,	/* jb +errout */
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
+	0x64, 0x48, 0x8b, 0x0c, 0x25,
+	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
+	0x48, 0x81, 0xc1,
+	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
+	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
+};
+
+/*
+ * void ptcfree(void *buf);
+ *
+ * if (buf == NULL)
+ * 	return;
+ *
+ * malloc_data_t *tag = buf;
+ * tag--;
+ * int size = tag->malloc_size;
+ * int tagval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
+ * if (tagval == MALLOC_SECOND_MAGIC) {
+ * 	tag--;
+ * } else if (tagval != MALLOC_MAGIC) {
+ * 	goto tofree;
+ * }
+ *
+ * if (size > cache_max)
+ * 	goto tofree;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_FRINI_JDONE	0x05
+#define	PTC_FRINI_JFREE	0x25
+#define	PTC_FRINI_MCS	0x30
+#define	PTC_FRINI_JOV	0x36
+#define	PTC_FRINI_SOFF	0x46
+static const uint8_t freeinit[] = {
+	0x48, 0x85, 0xff,		/* testq %rdi,%rdi */
+	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* jmp $JDONE (done) */
+	0x8b, 0x77, 0xf8,		/* movl -0x8(%rdi),%esi */
+	0x8b, 0x47, 0xfc,		/* movl -0x4(%rdi),%eax */
+	0x01, 0xf0,			/* addl %esi,%eax */
+	0x3d, 0x00, 0x70, 0xba, 0x16,	/* cmpl $MALLOC_2_MAGIC, %eax */
+	0x75, 0x06,			/* jne +0x6 (checkover) */
+	0x48, 0x8d, 0x47, 0xf0,		/* leaq -0x10(%rdi),%eax */
+	0xeb, 0x0f,			/* jmp +0xf (freebuf) */
+	0x3d, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl $MALLOC_MAGIC, %eax */
+	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jmp +JFREE (goto torfree) */
+	0x48, 0x8d, 0x47, 0xf8,		/* leaq -0x8(%rdi),%rax */
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +errout */
+	0x64, 0x48, 0x8b, 0x0c, 0x25,
+	0x00, 0x00, 0x00, 0x00,		/* movq %fs:0x0,%rcx */
+	0x48, 0x81, 0xc1,
+	0x00, 0x00, 0x00, 0x00,		/* addq $SOFF, %rcx */
+	0x48, 0x8d, 0x51, 0x08,		/* leaq 0x8(%rcx),%rdx */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ * } else ...				! goto next cache
+ */
+#define	PTC_INICACHE_CMP	0x03
+#define	PTC_INICACHE_SIZE	0x0c
+#define	PTC_INICACHE_JMP	0x11
+static const uint8_t inicache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x0c,			/* ja +0xc (next cache) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0xe9, 0x00, 0x00, 0x00, 0x00,	/* jmp $JMP (allocbuf) */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else ...				! goto next cache
+ */
+#define	PTC_GENCACHE_CMP	0x03
+#define	PTC_GENCACHE_SIZE	0x0c
+#define	PTC_GENCACHE_NUM	0x13
+#define	PTC_GENCACHE_JMP	0x18
+static const uint8_t gencache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x14,			/* ja +0xc (next cache) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0x48, 0x81, 0xc2,
+	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp +$JMP (allocbuf ) */
+};
+
+/*
+ * else if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else {
+ *	goto tofunc; 			! goto tomalloc if ptcmalloc.
+ * }					! goto tofree if ptcfree.
+ */
+#define	PTC_FINCACHE_CMP	0x03
+#define	PTC_FINCACHE_JMP	0x08
+#define	PTC_FINCACHE_SIZE	0x0c
+#define	PTC_FINCACHE_NUM	0x13
+static const uint8_t fincache[] = {
+	0x48, 0x81, 0xfe,
+	0x00, 0x00, 0x00, 0x00,		/* cmpq sizeof ($CACHE), %rsi */
+	0x77, 0x00,			/* ja +JMP (to real malloc) */
+	0x49, 0xc7, 0xc0,
+	0x00, 0x00, 0x00, 0x00,		/* movq sizeof ($CACHE), %r8 */
+	0x48, 0x81, 0xc2,
+	0x00, 0x00, 0x00, 0x00,		/* addq $8*ii, %rdx */
+
+};
+
+/*
+ * if (*root == NULL)
+ * 	goto tomalloc;
+ *
+ * malloc_data_t *ret = *root;
+ * *root = *(void **)ret;
+ * t->tm_size += csize;
+ * ret->malloc_size = size;
+ *
+ * if (size > UMEM_SECOND_ALIGN) {
+ *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ *	ret += 2;
+ * } else {
+ *	ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ *	ret += 1;
+ * }
+ *
+ * return ((void *)ret);
+ * tomalloc:
+ * 	return (malloc(orig_size));
+ */
+#define	PTC_MALFINI_ALLABEL	0x00
+#define	PTC_MALFINI_JMLABEL	0x40
+#define	PTC_MALFINI_JMADDR	0x41
+static const uint8_t malfini[] = {
+	0x48, 0x8b, 0x02,		/* movl (%rdx),%rax */
+	0x48, 0x85, 0xc0,		/* testq %rax,%rax */
+	0x74, 0x38,			/* je +0x38 (errout) */
+	0x4c, 0x8b, 0x08,		/* movq (%rax),%r9 */
+	0x4c, 0x89, 0x0a,		/* movq %r9,(%rdx) */
+	0x4c, 0x29, 0x01,		/* subq %rsi,(%rcx) */
+	0x48, 0x83, 0xfe, 0x10,		/* cmpq $0x10,%rsi */
+	0x76, 0x15,			/* jbe +0x15 */
+	0x41, 0xb9, 0x00, 0x70, 0xba, 0x16, /* movl $MALLOC_MAGIC_2, %r9d */
+	0x89, 0x70, 0x08,		/* movl %r9d,0x8(%rax) */
+	0x41, 0x29, 0xf1,		/* subl %esi, %r9d */
+	0x44, 0x89, 0x48, 0x0c,		/* movl %r9d, 0xc(%rax) */
+	0x48, 0x83, 0xc0, 0x10,		/* addq $0x10, %rax */
+	0xc3,				/* ret */
+	0x41, 0xb9, 0x00, 0xc0, 0x10, 0x3a,	/* movl %MALLOC_MAGIC, %r9d */
+	0x89, 0x30,			/* movl %esi,(%rax) */
+	0x41, 0x29, 0xf1,		/* subl %esi,%r9d */
+	0x44, 0x89, 0x48, 0x04,		/* movl %r9d,0x4(%rax) */
+	0x48, 0x83, 0xc0, 0x08,		/* addq $0x8,%rax */
+	0xc3,				/* ret */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $MALLOC */
+};
+
+/*
+ * if (t->tm_size + csize > umem_ptc_size)
+ * 	goto tofree;
+ *
+ * t->tm_size += csize
+ * *(void **)tag = *root;
+ * *root = tag;
+ * return;
+ * tofree:
+ * 	free(buf);
+ * 	return;
+ */
+#define	PTC_FRFINI_RBUFLABEL	0x00
+#define	PTC_FRFINI_CACHEMAX	0x09
+#define	PTC_FRFINI_DONELABEL	0x1b
+#define	PTC_FRFINI_JFLABEL	0x1c
+#define	PTC_FRFINI_JFADDR	0x1d
+static const uint8_t freefini[] = {
+	0x4c, 0x8b, 0x09,		/* movq (%rcx),%r9 */
+	0x4d, 0x01, 0xc1,		/* addq %r8, %r9 */
+	0x49, 0x81, 0xf9,
+	0x00, 0x00, 0x00, 0x00,		/* cmpl $THR_CACHE_MAX, %r9 */
+	0x77, 0x0d,			/* jae +0xd (torfree) */
+	0x4c, 0x01, 0x01,		/* addq %r8,(%rcx) */
+	0x4c, 0x8b, 0x0a,		/* movq (%rdx),%r9 */
+	0x4c, 0x89, 0x08,		/* movq %r9,(%rax) */
+	0x48, 0x89, 0x02,		/* movq %rax,(%rdx) */
+	0xc3,				/* ret */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp free */
+};
+
+/*
+ * Construct the initial part of malloc. off contains the offset from curthread
+ * to the root of the tmem structure. ep is the address of the label to error
+ * and jump to free. csize is the size of the largest umem_cache in ptcumem.
+ */
+static int
+genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
+{
+	uint32_t addr;
+
+	bcopy(malinit, bp, sizeof (malinit));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
+	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
+	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
+	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));
+
+	return (sizeof (malinit));
+}
+
+static int
+genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mcs)
+{
+	uint32_t addr;
+
+	bcopy(freeinit, bp, sizeof (freeinit));
+	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
+	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
+	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
+	bcopy(&mcs, bp + PTC_FRINI_MCS, sizeof (mcs));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
+	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
+	return (sizeof (freeinit));
+}
+
+
+/*
+ * Create the initial cache entry of the specified size. The value of ap tells
+ * us what the address of the label to try and allocate a buffer. This value is
+ * an offset from the current base to that value.
+ */
+static int
+genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+
+	bcopy(inicache, bp, sizeof (inicache));
+	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
+	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
+	ASSERT(addr != 0);
+	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));
+
+	return (sizeof (inicache));
+}
+
+static int
+genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+	uint32_t coff;
+
+	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
+	ASSERT(num != 0);
+	bcopy(gencache, bp, sizeof (gencache));
+	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
+	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
+	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));
+
+	return (sizeof (gencache));
+}
+
+static int
+genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
+{
+	uint8_t eap;
+	uint32_t coff;
+
+	ASSERT(ep <= 0xff && ep > 7);
+	ASSERT(UINT32_MAX / PTC_ROOT_SIZE > num);
+	bcopy(fincache, bp, sizeof (fincache));
+	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_FINCACHE_NUM, sizeof (coff));
+	eap = ep - PTC_FINCACHE_JMP - 1;
+	bcopy(&eap, bp + PTC_FINCACHE_JMP, sizeof (eap));
+
+	return (sizeof (fincache));
+}
+
+static int
+genasm_malfini(uint8_t *bp, uintptr_t mptr)
+{
+	uint32_t addr;
+
+	bcopy(malfini, bp, sizeof (malfini));
+	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
+	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));
+
+	return (sizeof (malfini));
+}
+
+static int
+genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
+{
+	uint32_t addr;
+
+	bcopy(freefini, bp, sizeof (freefini));
+	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
+	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
+	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));
+
+	return (sizeof (freefini));
+}
+
+/*
+ * The malloc inline assembly is constructed as follows:
+ *
+ * o Malloc prologue assembly
+ * o Generic first-cache check
+ * o n Generic cache checks (where n = _tmem_get_entries() - 2)
+ * o Generic last-cache check
+ * o Malloc epilogue assembly
+ *
+ * Generally there are at least three caches. When there is only one cache we
+ * only use the generic last-cache. In the case where there are two caches, we
+ * just leave out the middle ones.
+ */
+static int
+genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	int ii, off;
+	uint8_t *bp;
+	size_t total;
+	uint32_t allocoff, erroff;
+
+	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);
+
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
+	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;
+
+	bp = base;
+
+	off = genasm_malinit(bp, umem_tmem_off, erroff,
+	    umem_alloc_sizes[nents-1]);
+	bp += off;
+	allocoff -= off;
+	erroff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_malfini(bp, umem_genasm_omptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+static int
+genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	uint8_t *bp;
+	int ii, off;
+	size_t total;
+	uint32_t rbufoff, retoff, erroff;
+
+	/* Assume that nents has already been audited for us */
+	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
+	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
+	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);
+
+	bp = base;
+
+	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
+	    umem_alloc_sizes[nents - 1]);
+	bp += off;
+	erroff -= off;
+	rbufoff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
+		bp += off;
+		erroff -= off;
+		rbufoff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
+		bp += off;
+		rbufoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+/*ARGSUSED*/
+int
+umem_genasm(int *cp, umem_cache_t **caches, int nc)
+{
+	int nents, i;
+	uint8_t *mptr;
+	uint8_t *fptr;
+	uint32_t *ptr;
+	uint64_t v, *vptr;
+
+	mptr = (void *)((uintptr_t)&umem_genasm_mptr + 5);
+	fptr = (void *)((uintptr_t)&umem_genasm_fptr + 5);
+	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
+	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
+		return (1);
+
+	/*
+	 * The total number of caches that we can service is the minimum of:
+	 *  o the amount supported by libc
+	 *  o the total number of umem caches
+	 *  o we use a single byte addl, so its MAX_UINT32 / sizeof (uintptr_t).
+	 *    For 64-bit, this is MAX_UINT32 >> 3, a lot.
+	 */
+	nents = _tmem_get_nentries();
+
+	if (UMEM_GENASM_MAX64 < nents)
+		nents = UMEM_GENASM_MAX64;
+
+	if (nc < nents)
+		nents = nc;
+
+	/* Based on our constraints, this is not an error */
+	if (nents == 0 || umem_ptc_size == 0)
+		return (0);
+
+	/* Grab the original malloc and free locations */
+	ptr = (void *)(mptr - 4);
+	umem_genasm_omptr = *ptr + (uintptr_t)mptr;
+	ptr = (void *)(fptr - 4);
+	umem_genasm_ofptr = *ptr + (uintptr_t)fptr;
+
+	/* Take into account the jump */
+	if (genasm_malloc(mptr, umem_genasm_fsize - 5, nents, cp) != 0)
+		return (1);
+
+	if (genasm_free(fptr, umem_genasm_fsize - 5, nents, cp) != 0)
+		return (1);
+
+	/* nop out the jump with a multibyte jump */
+	vptr = (void *)&umem_genasm_mptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+	vptr = (void *)&umem_genasm_fptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+
+
+	for (i = 0; i < nents; i++)
+		caches[i]->cache_flags |= UMF_PTC;
+
+	return (0);
+}

atomic.h

@@ -0,0 +1,438 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_SYS_ATOMIC_H
+#define	_SYS_ATOMIC_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <sys/types.h>
+#include <sys/inttypes.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
+	(defined(__i386) || defined(__amd64))
+#include <asm/atomic.h>
+#endif
+
+#if defined(_KERNEL) || defined(__STDC__)
+/*
+ * Increment target.
+ */
+extern void atomic_inc_8(volatile uint8_t *);
+extern void atomic_inc_uchar(volatile uchar_t *);
+extern void atomic_inc_16(volatile uint16_t *);
+extern void atomic_inc_ushort(volatile ushort_t *);
+extern void atomic_inc_32(volatile uint32_t *);
+extern void atomic_inc_uint(volatile uint_t *);
+extern void atomic_inc_ulong(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_inc_64(volatile uint64_t *);
+#endif
+
+/*
+ * Decrement target
+ */
+extern void atomic_dec_8(volatile uint8_t *);
+extern void atomic_dec_uchar(volatile uchar_t *);
+extern void atomic_dec_16(volatile uint16_t *);
+extern void atomic_dec_ushort(volatile ushort_t *);
+extern void atomic_dec_32(volatile uint32_t *);
+extern void atomic_dec_uint(volatile uint_t *);
+extern void atomic_dec_ulong(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_dec_64(volatile uint64_t *);
+#endif
+
+/*
+ * Add delta to target
+ */
+extern void atomic_add_8(volatile uint8_t *, int8_t);
+extern void atomic_add_char(volatile uchar_t *, signed char);
+extern void atomic_add_16(volatile uint16_t *, int16_t);
+extern void atomic_add_short(volatile ushort_t *, short);
+extern void atomic_add_32(volatile uint32_t *, int32_t);
+extern void atomic_add_int(volatile uint_t *, int);
+extern void atomic_add_ptr(volatile void *, ssize_t);
+extern void atomic_add_long(volatile ulong_t *, long);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_add_64(volatile uint64_t *, int64_t);
+#endif
+
+/*
+ * logical OR bits with target
+ */
+extern void atomic_or_8(volatile uint8_t *, uint8_t);
+extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
+extern void atomic_or_16(volatile uint16_t *, uint16_t);
+extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
+extern void atomic_or_32(volatile uint32_t *, uint32_t);
+extern void atomic_or_uint(volatile uint_t *, uint_t);
+extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_or_64(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * logical AND bits with target
+ */
+extern void atomic_and_8(volatile uint8_t *, uint8_t);
+extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
+extern void atomic_and_16(volatile uint16_t *, uint16_t);
+extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
+extern void atomic_and_32(volatile uint32_t *, uint32_t);
+extern void atomic_and_uint(volatile uint_t *, uint_t);
+extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern void atomic_and_64(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * As above, but return the new value.  Note that these _nv() variants are
+ * substantially more expensive on some platforms than the no-return-value
+ * versions above, so don't use them unless you really need to know the
+ * new value *atomically* (e.g. when decrementing a reference count and
+ * checking whether it went to zero).
+ */
+
+/*
+ * Increment target and return new value.
+ */
+extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
+extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
+extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
+extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
+extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
+extern uint_t atomic_inc_uint_nv(volatile uint_t *);
+extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
+#endif
+
+/*
+ * Decrement target and return new value.
+ */
+extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
+extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
+extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
+extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
+extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
+extern uint_t atomic_dec_uint_nv(volatile uint_t *);
+extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
+#endif
+
+/*
+ * Add delta to target
+ */
+extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
+extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
+extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
+extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
+extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
+extern uint_t atomic_add_int_nv(volatile uint_t *, int);
+extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
+extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
+#endif
+
+/*
+ * logical OR bits with target and return new value.
+ */
+extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
+extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
+extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
+extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
+extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
+extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
+extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * logical AND bits with target and return new value.
+ */
+extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
+extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
+extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
+extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
+extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
+extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
+extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * If *arg1 == arg2, set *arg1 = arg3; return old value
+ */
+extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
+extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
+extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
+extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
+extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
+extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
+extern void *atomic_cas_ptr(volatile void *, void *, void *);
+extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
+#endif
+
+/*
+ * Swap target and return old value
+ */
+extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
+extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
+extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
+extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
+extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
+extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
+extern void *atomic_swap_ptr(volatile void *, void *);
+extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
+#if defined(_KERNEL) || defined(_INT64_TYPE)
+extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
+#endif
+
+/*
+ * Perform an exclusive atomic bit set/clear on a target.
+ * Returns 0 if bit was sucessfully set/cleared, or -1
+ * if the bit was already set/cleared.
+ */
+extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
+extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
+
+/*
+ * Generic memory barrier used during lock entry, placed after the
+ * memory operation that acquires the lock to guarantee that the lock
+ * protects its data.  No stores from after the memory barrier will
+ * reach visibility, and no loads from after the barrier will be
+ * resolved, before the lock acquisition reaches global visibility.
+ */
+extern void membar_enter(void);
+
+/*
+ * Generic memory barrier used during lock exit, placed before the
+ * memory operation that releases the lock to guarantee that the lock
+ * protects its data.  All loads and stores issued before the barrier
+ * will be resolved before the subsequent lock update reaches visibility.
+ */
+extern void membar_exit(void);
+
+/*
+ * Arrange that all stores issued before this point in the code reach
+ * global visibility before any stores that follow; useful in producer
+ * modules that update a data item, then set a flag that it is available.
+ * The memory barrier guarantees that the available flag is not visible
+ * earlier than the updated data, i.e. it imposes store ordering.
+ */
+extern void membar_producer(void);
+
+/*
+ * Arrange that all loads issued before this point in the code are
+ * completed before any subsequent loads; useful in consumer modules
+ * that check to see if data is available and read the data.
+ * The memory barrier guarantees that the data is not sampled until
+ * after the available flag has been seen, i.e. it imposes load ordering.
+ */
+extern void membar_consumer(void);
+#endif
+
+#if !defined(_KERNEL) && !defined(__STDC__)
+extern void atomic_inc_8();
+extern void atomic_inc_uchar();
+extern void atomic_inc_16();
+extern void atomic_inc_ushort();
+extern void atomic_inc_32();
+extern void atomic_inc_uint();
+extern void atomic_inc_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_inc_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_dec_8();
+extern void atomic_dec_uchar();
+extern void atomic_dec_16();
+extern void atomic_dec_ushort();
+extern void atomic_dec_32();
+extern void atomic_dec_uint();
+extern void atomic_dec_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_dec_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_add_8();
+extern void atomic_add_char();
+extern void atomic_add_16();
+extern void atomic_add_short();
+extern void atomic_add_32();
+extern void atomic_add_int();
+extern void atomic_add_ptr();
+extern void atomic_add_long();
+#if defined(_INT64_TYPE)
+extern void atomic_add_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_or_8();
+extern void atomic_or_uchar();
+extern void atomic_or_16();
+extern void atomic_or_ushort();
+extern void atomic_or_32();
+extern void atomic_or_uint();
+extern void atomic_or_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_or_64();
+#endif /* defined(_INT64_TYPE) */
+extern void atomic_and_8();
+extern void atomic_and_uchar();
+extern void atomic_and_16();
+extern void atomic_and_ushort();
+extern void atomic_and_32();
+extern void atomic_and_uint();
+extern void atomic_and_ulong();
+#if defined(_INT64_TYPE)
+extern void atomic_and_64();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_inc_8_nv();
+extern uchar_t atomic_inc_uchar_nv();
+extern uint16_t atomic_inc_16_nv();
+extern ushort_t atomic_inc_ushort_nv();
+extern uint32_t atomic_inc_32_nv();
+extern uint_t atomic_inc_uint_nv();
+extern ulong_t atomic_inc_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_inc_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_dec_8_nv();
+extern uchar_t atomic_dec_uchar_nv();
+extern uint16_t atomic_dec_16_nv();
+extern ushort_t atomic_dec_ushort_nv();
+extern uint32_t atomic_dec_32_nv();
+extern uint_t atomic_dec_uint_nv();
+extern ulong_t atomic_dec_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_dec_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_add_8_nv();
+extern uchar_t atomic_add_char_nv();
+extern uint16_t atomic_add_16_nv();
+extern ushort_t atomic_add_short_nv();
+extern uint32_t atomic_add_32_nv();
+extern uint_t atomic_add_int_nv();
+extern void *atomic_add_ptr_nv();
+extern ulong_t atomic_add_long_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_add_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_or_8_nv();
+extern uchar_t atomic_or_uchar_nv();
+extern uint16_t atomic_or_16_nv();
+extern ushort_t atomic_or_ushort_nv();
+extern uint32_t atomic_or_32_nv();
+extern uint_t atomic_or_uint_nv();
+extern ulong_t atomic_or_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_or_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_and_8_nv();
+extern uchar_t atomic_and_uchar_nv();
+extern uint16_t atomic_and_16_nv();
+extern ushort_t atomic_and_ushort_nv();
+extern uint32_t atomic_and_32_nv();
+extern uint_t atomic_and_uint_nv();
+extern ulong_t atomic_and_ulong_nv();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_and_64_nv();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_cas_8();
+extern uchar_t atomic_cas_uchar();
+extern uint16_t atomic_cas_16();
+extern ushort_t atomic_cas_ushort();
+extern uint32_t atomic_cas_32();
+extern uint_t atomic_cas_uint();
+extern void *atomic_cas_ptr();
+extern ulong_t atomic_cas_ulong();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_cas_64();
+#endif /* defined(_INT64_TYPE) */
+extern uint8_t atomic_swap_8();
+extern uchar_t atomic_swap_uchar();
+extern uint16_t atomic_swap_16();
+extern ushort_t atomic_swap_ushort();
+extern uint32_t atomic_swap_32();
+extern uint_t atomic_swap_uint();
+extern void *atomic_swap_ptr();
+extern ulong_t atomic_swap_ulong();
+#if defined(_INT64_TYPE)
+extern uint64_t atomic_swap_64();
+#endif /* defined(_INT64_TYPE) */
+
+
+extern int atomic_set_long_excl();
+extern int atomic_clear_long_excl();
+
+extern void membar_enter();
+extern void membar_exit();
+extern void membar_producer();
+extern void membar_consumer();
+
+#endif
+
+#if defined(_KERNEL)
+
+#if defined(_LP64) || defined(_ILP32)
+#define	atomic_add_ip		atomic_add_long
+#define	atomic_add_ip_nv	atomic_add_long_nv
+#define	casip			atomic_cas_ulong
+#endif
+
+#if defined(__sparc)
+extern uint8_t ldstub(uint8_t *);
+#endif
+
+/*
+ * Legacy kernel interfaces; they will go away (eventually).
+ */
+extern uint8_t cas8(uint8_t *, uint8_t, uint8_t);
+extern uint32_t cas32(uint32_t *, uint32_t, uint32_t);
+extern uint64_t cas64(uint64_t *, uint64_t, uint64_t);
+extern ulong_t caslong(ulong_t *, ulong_t, ulong_t);
+extern void *casptr(void *, void *, void *);
+extern void atomic_and_long(ulong_t *, ulong_t);
+extern void atomic_or_long(ulong_t *, ulong_t);
+#if defined(__sparc)
+extern uint32_t swapl(uint32_t *, uint32_t);
+#endif
+
+#endif	/* _KERNEL */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_ATOMIC_H */

autogen.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-autoreconf -i -s
+autoreconf -fis

configure.ac

@@ -1,5 +1,6 @@
-AC_INIT([umem], [1.0.2], [], [umem])
-AM_INIT_AUTOMAKE([dist-bzip2])
+AC_INIT([umem], [1.0.3], [], [umem])
+AM_INIT_AUTOMAKE([dist-bzip2 subdir-objects])
+AC_CONFIG_MACRO_DIR([m4])
 
 AC_PROG_CC
 AM_PROG_AS
@@ -8,8 +9,8 @@ AC_PROG_LIBTOOL
 AC_C_INLINE
 
 AC_MSG_CHECKING([whether pthread_mutex_t is larger than 24 bytes])
-AC_TRY_RUN( 
-  [   
+AC_TRY_RUN(
+  [
 #include <pthread.h>
 int main(void){return (sizeof(pthread_mutex_t) > 24);}
   ],
@@ -19,7 +20,7 @@ int main(void){return (sizeof(pthread_mutex_t) > 24);}
     AC_DEFINE(UMEM_PTHREAD_MUTEX_TOO_BIG, [1], [need bigger cache])
     AC_MSG_WARN([*** increasing umem cpu cache size to compensate.])
   ]
-) 
+)
 
 AC_CHECK_LIB(dl,dlopen)
 

envvar.c

@@ -23,9 +23,9 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)envvar.c	1.5	05/06/08 SMI" */
@@ -181,7 +181,10 @@ static umem_env_item_t umem_options_items[] = {
 	},
 #endif
 #endif
-
+	{ "perthread_cache",	"Evolving",	ITEM_SIZE,
+		"Size (in bytes) of per-thread allocation cache",
+		NULL, 0, NULL, &umem_ptc_size
+	},
 	{ NULL, "-- end of UMEM_OPTIONS --",	ITEM_INVALID }
 };
 

getpcstack.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,12 +18,12 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- */
-/*
- * Portions Copyright 2006-2008 Message Systems, Inc.
+ *
+ * Copyright 2006-2008 Message Systems, Inc.
  */
 
 /* #pragma ident	"@(#)getpcstack.c	1.5	05/06/08 SMI" */

i386/asm_subr.s

@@ -0,0 +1,72 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <sys/asm_linkage.h>
+
+#if defined(lint)
+
+void *
+getfp(void)
+{
+	return (NULL);
+}
+
+#ifndef UMEM_STANDALONE
+void
+_breakpoint(void)
+{
+	return;
+}
+#endif
+
+#else	/* lint */
+
+#if defined(__amd64)
+
+	ENTRY(getfp)
+	movq	%rbp, %rax
+	ret
+	SET_SIZE(getfp)
+
+#else	/* __i386 */
+
+	ENTRY(getfp)
+	movl	%ebp, %eax
+	ret
+	SET_SIZE(getfp)
+
+#endif
+
+#ifndef UMEM_STANDALONE
+	ENTRY(_breakpoint)
+	int	$3
+	ret
+	SET_SIZE(_breakpoint)
+#endif
+
+#endif	/* lint */

i386/atomic.h

@@ -0,0 +1,104 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _ASM_ATOMIC_H
+#define	_ASM_ATOMIC_H
+
+#include <sys/ccompile.h>
+#include <sys/types.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if !defined(__lint) && defined(__GNUC__)
+
+#if defined(__amd64)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andq %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#ifdef notdef
+extern __GNU_INLINE uint64_t
+cas64(uint64_t *target, uint64_t cmp,
+	uint64_t newval)
+{
+	uint64_t retval;
+
+	__asm__ __volatile__(
+	    "movq %2, %%rax; lock; cmpxchgq %3, (%1)"
+	    : "=a" (retval)
+	    : "r" (target), "r" (cmp), "r" (newval));
+	return (retval);
+}
+#endif
+
+#elif defined(__i386)
+
+extern __GNU_INLINE void
+atomic_or_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; orl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+extern __GNU_INLINE void
+atomic_and_long(ulong_t *target, ulong_t bits)
+{
+	__asm__ __volatile__(
+	    "lock; andl %1, (%0)"
+	    : /* no output */
+	    : "r" (target), "r" (bits));
+}
+
+#else
+#error	"port me"
+#endif
+
+#endif /* !__lint && __GNUC__ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _ASM_ATOMIC_H */

i386/umem_genasm.c

@@ -0,0 +1,603 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2012 Joyent, Inc.  All rights reserved.
+ */
+
+/*
+ * Don't Panic! If you find the blocks of assembly that follow confusing and
+ * you're questioning why they exist, please go read section 8 of the umem.c big
+ * theory statement. Next familiarize yourself with the malloc and free
+ * implementations in libumem's malloc.c.
+ *
+ * What follows is the i386 implementation of the thread caching automatic
+ * assembly generation. With i386 a function only has three registers its
+ * allowed to change without restoring them: eax, ecx, and edx. All others have
+ * to be preserved. Since the set of registers we have available is so small, we
+ * have to make use of esi, ebx, and edi and save their original values to the
+ * stack.
+ *
+ * Malloc register usage:
+ * 	o. esi: Size of the malloc (passed into us and modified)
+ * 	o. edi: Size of the cache
+ * 	o. eax: Buffer to return
+ * 	o. ebx: Scratch space and temporary values
+ * 	o. ecx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. edx: Pointer to the tmem_t array of roots
+ *
+ * Free register usage:
+ * 	o. esi: Size of the malloc (passed into us and modified)
+ * 	o. edi: Size of the cache
+ * 	o. eax: Buffer to free
+ * 	o. ebx: Scratch space and temporary values
+ * 	o. ecx: Pointer to the tmem_t in the ulwp_t.
+ * 	o. edx: Pointer to the tmem_t array of roots
+ *
+ * Once we determine what cache we are using, we increment %edx to the
+ * appropriate offset and set %edi with the size of the cache. This means that
+ * when we break out to the normal buffer allocation point %edx contains the
+ * head of the linked list and %edi is the amount that we have to adjust the
+ * total amount cached by the thread.
+ *
+ * Each block of assembly has psuedocode that describes its purpose.
+ */
+
+#include <inttypes.h>
+#include <strings.h>
+#include <umem_impl.h>
+#include "umem_base.h"
+
+#include <atomic.h>
+
+int umem_genasm_supported = 1;
+uintptr_t umem_genasm_mptr;
+size_t umem_genasm_msize;
+uintptr_t umem_genasm_fptr;
+size_t umem_genasm_fsize;
+static uintptr_t umem_genasm_omptr;
+static uintptr_t umem_genasm_ofptr;
+
+/*
+ * The maximum number of caches we can support. We use a single byte addl so
+ * this is 255 (UINT8_MAX) / sizeof (uintptr_t). In this case 63
+ */
+#define	UMEM_GENASM_MAX32	63
+
+#define	PTC_JMPADDR(dest, src)	(dest - (src + 4))
+#define	PTC_ROOT_SIZE	sizeof (uintptr_t)
+#define	MULTINOP	 0x0000441f0f
+
+/*
+ * void *ptcmalloc(size_t orig_size);
+ *
+ * size_t size = orig_size + 8;
+ *
+ * if (size < orig_size)
+ * 	goto tomalloc;		! This is overflow
+ *
+ * if (size > cache_size)
+ * 	goto tomalloc;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_MALINIT_JOUT	0x0e
+#define	PTC_MALINIT_MCS	0x14
+#define	PTC_MALINIT_JOV	0x1a
+#define	PTC_MALINIT_SOFF	0x27
+static const uint8_t malinit[] = {
+	0x55,					/* pushl %ebp */
+	0x89, 0xe5,				/* movl %esp, %ebp */
+	0x57,					/* pushl %edi */
+	0x56,					/* pushl %esi */
+	0x53,					/* pushl %ebx */
+	0x8b, 0x75, 0x08,			/* movl 0x8(%ebp), %esi */
+	0x83, 0xc6, 0x08,			/* addl $0x8,%esi */
+	0x0f, 0x82, 0x00, 0x00, 0x00, 0x00, 	/* jc +$JMP (errout) */
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +$JMP (errout) */
+	0x65, 0x8b, 0x0d, 0x00, 0x0, 0x00, 0x00, 	/* movl %gs:0x0,%ecx */
+	0x81, 0xc1, 0x00, 0x00,	0x00, 0x00, 	/* addl $OFF, %ecx */
+	0x8d, 0x51, 0x04			/* leal 0x4(%ecx), %edx */
+};
+
+/*
+ * void ptcfree(void *buf);
+ *
+ * if (buf == NULL)
+ * 	return;
+ *
+ * malloc_data_t *tag = buf;
+ * tag--;
+ * int size = tag->malloc_size;
+ * int tagtval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
+ *
+ * if (tagval != MALLOC_MAGIC)
+ * 	goto tofree;
+ *
+ * if (size > cache_max)
+ * 	goto tofree;
+ *
+ * tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
+ * void **roots = t->tm_roots;
+ */
+#define	PTC_FRINI_JDONE	0x0d
+#define	PTC_FRINI_JFREE	0x23
+#define	PTC_FRINI_MCS	0x29
+#define	PTC_FRINI_JOV	0x2f
+#define	PTC_FRINI_SOFF	0x3c
+static const uint8_t freeinit[] = {
+	0x55,					/* pushl %ebp */
+	0x89, 0xe5,				/* movl %esp, %ebp */
+	0x57,					/* pushl %edi */
+	0x56,					/* pushl %esi */
+	0x53,					/* pushl %ebx */
+	0x8b, 0x45, 0x08,			/* movl 0x8(%ebp), %eax */
+	0x85, 0xc0,				/* testl %eax, %eax */
+	0x0f, 0x84, 0x00, 0x00, 0x00, 0x00,	/* je $JDONE (done) */
+	0x83, 0xe8, 0x08,			/* subl $0x8,%eax */
+	0x8b, 0x30,				/* movl (%eax),%esi */
+	0x8b, 0x50, 0x04,			/* movl 0x4(%eax),%edx */
+	0x01, 0xf2,				/* addl %esi,%edx */
+	0x81, 0xfa, 0x00, 0xc0, 0x10, 0x3a,	/* cmpl MAGIC32, %edx */
+	0x0f, 0x85, 0x00, 0x00, 0x00, 0x00,	/* jne +JFREE (goto freebuf) */
+
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x0f, 0x87, 0x00, 0x00, 0x00, 0x00,	/* ja +$JMP (errout) */
+	0x65, 0x8b, 0x0d, 0x00, 0x0, 0x00, 0x00, /* movl %gs:0x0,%ecx */
+	0x81, 0xc1, 0x00, 0x00,	0x00, 0x00,	/* addl $0xOFF, %ecx */
+	0x8d, 0x51, 0x04			/* leal 0x4(%ecx),%edx */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ * } else ...				! goto next cache
+ */
+#define	PTC_INICACHE_CMP	0x02
+#define	PTC_INICACHE_SIZE 0x09
+#define	PTC_INICACHE_JMP	0x0e
+static const uint8_t inicache[] = {
+	0x81, 0xfe, 0xff, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($C0), %esi */
+	0x77, 0x0a,				/* ja +0xa */
+	0xbf, 0xff, 0x00, 0x00, 0x00, 		/* movl sizeof ($C0), %edi */
+	0xe9, 0x00, 0x00, 0x00, 0x00		/* jmp +$JMP (allocbuf) */
+};
+
+/*
+ * if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else ...				! goto next cache
+ */
+#define	PTC_GENCACHE_CMP	0x02
+#define	PTC_GENCACHE_NUM	0x0a
+#define	PTC_GENCACHE_SIZE 0x0c
+#define	PTC_GENCACHE_JMP	0x11
+static const uint8_t gencache[] = {
+	0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, 	/* cmpl sizeof ($CACHE), %esi */
+	0x77, 0x0d,				/* ja +0xd (next cache) */
+	0x83, 0xc2, 0x00,			/* addl $4*$ii, %edx */
+	0xbf, 0x00, 0x00, 0x00, 0x00, 		/* movl sizeof ($CACHE), %edi */
+	0xe9, 0x00, 0x00, 0x00, 0x00 		/* jmp +$JMP (allocbuf) */
+};
+
+/*
+ * else if (size <= $CACHE_SIZE) {
+ *	csize = $CACHE_SIZE;
+ *	roots += $CACHE_NUM;
+ * } else {
+ *	goto tofunc; 			! goto tomalloc if ptcmalloc.
+ * }					! goto tofree if ptcfree.
+ */
+#define	PTC_FINCACHE_CMP 0x02
+#define	PTC_FINCACHE_JMP	0x07
+#define	PTC_FINCACHE_NUM 0x0a
+#define	PTC_FINCACHE_SIZE 0x0c
+static const uint8_t fincache[] = {
+	0x81, 0xfe, 0xff, 0x00, 0x00, 0x00,	/* cmpl sizeof ($CLAST), %esi */
+	0x77, 0x00,				/* ja +$JMP (to errout) */
+	0x83, 0xc2, 0x00,			/* addl $4*($NCACHES-1), %edx */
+	0xbf, 0x00, 0x00, 0x00, 0x00, 		/* movl sizeof ($CLAST), %edi */
+};
+
+/*
+ * if (*root == NULL)
+ * 	goto tomalloc;
+ *
+ * malloc_data_t *ret = *root;
+ * *root = *(void **)ret;
+ * t->tm_size += csize;
+ * ret->malloc_size = size;
+ *
+ * ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
+ * ret++;
+ *
+ * return ((void *)ret);
+ * tomalloc:
+ * 	return (malloc(orig_size));
+ */
+#define	PTC_MALFINI_ALLABEL	0x00
+#define	PTC_MALFINI_JMLABEL	0x20
+#define	PTC_MALFINI_JMADDR	0x25
+static const uint8_t malfini[] = {
+	/* allocbuf: */
+	0x8b, 0x02,			/* movl (%edx), %eax */
+	0x85, 0xc0,			/* testl %eax, %eax */
+	0x74, 0x1a,			/* je +0x1a (errout) */
+	0x8b, 0x18,			/* movl (%eax), %esi */
+	0x89, 0x1a,			/* movl %esi, (%edx) */
+	0x29, 0x39,			/* subl %edi, (%ecx) */
+	0x89, 0x30,			/* movl %esi, ($eax) */
+	0xba, 0x00, 0xc0, 0x10, 0x3a,	/* movl $0x3a10c000,%edx */
+	0x29, 0xf2,			/* subl %esi, %edx */
+	0x89, 0x50, 0x04,		/* movl %edx, 0x4(%eax) */
+	0x83, 0xc0, 0x08,		/* addl %0x8, %eax */
+	0x5b,				/* popl %ebx */
+	0x5e,				/* popl %esi */
+	0x5f,				/* popl %edi */
+	0xc9,				/* leave */
+	0xc3,				/* ret */
+	/* errout: */
+	0x5b,				/* popl %ebx */
+	0x5e,				/* popl %esi */
+	0x5f,				/* popl %edi */
+	0xc9,				/* leave */
+	0xe9, 0x00, 0x00, 0x00, 0x00	/* jmp $malloc */
+};
+
+/*
+ * if (t->tm_size + csize > umem_ptc_size)
+ * 	goto tofree;
+ *
+ * t->tm_size += csize
+ * *(void **)tag = *root;
+ * *root = tag;
+ * return;
+ * tofree:
+ * 	free(buf);
+ * 	return;
+ */
+#define	PTC_FRFINI_RBUFLABEL	0x00
+#define	PTC_FRFINI_CACHEMAX	0x06
+#define	PTC_FRFINI_DONELABEL	0x14
+#define	PTC_FRFINI_JFLABEL	0x19
+#define	PTC_FRFINI_JFADDR	0x1e
+static const uint8_t freefini[] = {
+	/* freebuf: */
+	0x8b, 0x19,				/* movl (%ecx),%ebx */
+	0x01, 0xfb,				/* addl %edi,%ebx */
+	0x81, 0xfb, 0x00, 0x00, 0x00, 0x00, 	/* cmpl maxsize, %ebx */
+	0x73, 0x0d,				/* jae +0xd <tofree> */
+	0x01, 0x39,				/* addl %edi,(%ecx) */
+	0x8b, 0x3a,				/* movl (%edx),%edi */
+	0x89, 0x38,				/* movl %edi,(%eax) */
+	0x89, 0x02,				/* movl %eax,(%edx) */
+	/* done: */
+	0x5b,					/* popl %ebx */
+	0x5e,					/* popl %esi */
+	0x5f,					/* popl %edi */
+	0xc9,					/* leave */
+	0xc3,					/* ret */
+	/* realfree: */
+	0x5b,					/* popl %ebx */
+	0x5e,					/* popl %esi */
+	0x5f,					/* popl %edi */
+	0xc9,					/* leave */
+	0xe9, 0x00, 0x00, 0x00, 0x00		/* jmp free */
+};
+
+/*
+ * Construct the initial part of malloc. off contains the offset from curthread
+ * to the root of the tmem structure. ep is the address of the label to error
+ * and jump to free. csize is the size of the largest umem_cache in ptcumem.
+ */
+static int
+genasm_malinit(uint8_t *bp, uint32_t off, uint32_t ep, uint32_t csize)
+{
+	uint32_t addr;
+
+	bcopy(malinit, bp, sizeof (malinit));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOUT);
+	bcopy(&addr, bp + PTC_MALINIT_JOUT, sizeof (addr));
+	bcopy(&csize, bp + PTC_MALINIT_MCS, sizeof (csize));
+	addr = PTC_JMPADDR(ep, PTC_MALINIT_JOV);
+	bcopy(&addr, bp + PTC_MALINIT_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_MALINIT_SOFF, sizeof (off));
+
+	return (sizeof (malinit));
+}
+
+static int
+genasm_frinit(uint8_t *bp, uint32_t off, uint32_t dp, uint32_t ep, uint32_t mc)
+{
+	uint32_t addr;
+
+	bcopy(freeinit, bp, sizeof (freeinit));
+	addr = PTC_JMPADDR(dp, PTC_FRINI_JDONE);
+	bcopy(&addr, bp + PTC_FRINI_JDONE, sizeof (addr));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JFREE);
+	bcopy(&addr, bp + PTC_FRINI_JFREE, sizeof (addr));
+	bcopy(&mc, bp + PTC_FRINI_MCS, sizeof (mc));
+	addr = PTC_JMPADDR(ep, PTC_FRINI_JOV);
+	bcopy(&addr, bp + PTC_FRINI_JOV, sizeof (addr));
+	bcopy(&off, bp + PTC_FRINI_SOFF, sizeof (off));
+	return (sizeof (freeinit));
+}
+
+/*
+ * Create the initial cache entry of the specified size. The value of ap tells
+ * us what the address of the label to try and allocate a buffer. This value is
+ * an offset from the current base to that value.
+ */
+static int
+genasm_firstcache(uint8_t *bp, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+
+	bcopy(inicache, bp, sizeof (inicache));
+	bcopy(&csize, bp + PTC_INICACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_INICACHE_SIZE, sizeof (csize));
+	addr = PTC_JMPADDR(ap, PTC_INICACHE_JMP);
+	ASSERT(addr != 0);
+	bcopy(&addr, bp + PTC_INICACHE_JMP, sizeof (addr));
+
+	return (sizeof (inicache));
+}
+
+static int
+genasm_gencache(uint8_t *bp, int num, uint32_t csize, uint32_t ap)
+{
+	uint32_t addr;
+	uint8_t	coff;
+
+	ASSERT(256 / PTC_ROOT_SIZE > num);
+	ASSERT(num != 0);
+	bcopy(gencache, bp, sizeof (gencache));
+	bcopy(&csize, bp + PTC_GENCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_GENCACHE_SIZE, sizeof (csize));
+	coff = num * PTC_ROOT_SIZE;
+	bcopy(&coff, bp + PTC_GENCACHE_NUM, sizeof (coff));
+	addr = PTC_JMPADDR(ap, PTC_GENCACHE_JMP);
+	bcopy(&addr, bp + PTC_GENCACHE_JMP, sizeof (addr));
+
+	return (sizeof (gencache));
+}
+
+static int
+genasm_lastcache(uint8_t *bp, int num, uint32_t csize, uint32_t ep)
+{
+	uint8_t addr;
+
+	ASSERT(ep <= 0xff && ep > 7);
+	ASSERT(256 / PTC_ROOT_SIZE > num);
+	bcopy(fincache, bp, sizeof (fincache));
+	bcopy(&csize, bp + PTC_FINCACHE_CMP, sizeof (csize));
+	bcopy(&csize, bp + PTC_FINCACHE_SIZE, sizeof (csize));
+	addr = num * PTC_ROOT_SIZE;
+	bcopy(&addr, bp + PTC_FINCACHE_NUM, sizeof (addr));
+	addr = ep - PTC_FINCACHE_JMP - 1;
+	bcopy(&addr, bp + PTC_FINCACHE_JMP, sizeof (addr));
+
+	return (sizeof (fincache));
+}
+
+static int
+genasm_malfini(uint8_t *bp, uintptr_t mptr)
+{
+	uint32_t addr;
+
+	bcopy(malfini, bp, sizeof (malfini));
+	addr = PTC_JMPADDR(mptr, ((uintptr_t)bp + PTC_MALFINI_JMADDR));
+	bcopy(&addr, bp + PTC_MALFINI_JMADDR, sizeof (addr));
+
+	return (sizeof (malfini));
+}
+
+static int
+genasm_frfini(uint8_t *bp, uint32_t maxthr, uintptr_t fptr)
+{
+	uint32_t addr;
+
+	bcopy(freefini, bp, sizeof (freefini));
+	bcopy(&maxthr, bp + PTC_FRFINI_CACHEMAX, sizeof (maxthr));
+	addr = PTC_JMPADDR(fptr, ((uintptr_t)bp + PTC_FRFINI_JFADDR));
+	bcopy(&addr, bp + PTC_FRFINI_JFADDR, sizeof (addr));
+
+	return (sizeof (freefini));
+}
+
+/*
+ * The malloc inline assembly is constructed as follows:
+ *
+ * o Malloc prologue assembly
+ * o Generic first-cache check
+ * o n Generic cache checks (where n = _tmem_get_entries() - 2)
+ * o Generic last-cache check
+ * o Malloc epilogue assembly
+ *
+ * Generally there are at least three caches. When there is only one cache we
+ * only use the generic last-cache. In the case where there are two caches, we
+ * just leave out the middle ones.
+ */
+static int
+genasm_malloc(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	int ii, off;
+	uint8_t *bp;
+	size_t total;
+	uint32_t allocoff, erroff;
+
+	total = sizeof (malinit) + sizeof (malfini) + sizeof (fincache);
+
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - sizeof (malfini) + PTC_MALFINI_JMLABEL;
+	allocoff = total - sizeof (malfini) + PTC_MALFINI_ALLABEL;
+
+	bp = base;
+
+	off = genasm_malinit(bp, umem_tmem_off, erroff,
+	    umem_alloc_sizes[nents-1]);
+	bp += off;
+	allocoff -= off;
+	erroff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], allocoff);
+		bp += off;
+		allocoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_malfini(bp, umem_genasm_omptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+static int
+genasm_free(void *base, size_t len, int nents, int *umem_alloc_sizes)
+{
+	uint8_t *bp;
+	int ii, off;
+	size_t total;
+	uint32_t rbufoff, retoff, erroff;
+
+	/* Assume that nents has already been audited for us */
+	total = sizeof (freeinit) + sizeof (freefini) + sizeof (fincache);
+	if (nents >= 2)
+		total += sizeof (inicache) + sizeof (gencache) * (nents - 2);
+
+	if (total > len)
+		return (1);
+
+	erroff = total - (sizeof (freefini) - PTC_FRFINI_JFLABEL);
+	rbufoff = total - (sizeof (freefini) - PTC_FRFINI_RBUFLABEL);
+	retoff = total - (sizeof (freefini) - PTC_FRFINI_DONELABEL);
+
+	bp = base;
+
+	off = genasm_frinit(bp, umem_tmem_off, retoff, erroff,
+	    umem_alloc_sizes[nents - 1]);
+	bp += off;
+	erroff -= off;
+	rbufoff -= off;
+
+	if (nents > 1) {
+		off = genasm_firstcache(bp, umem_alloc_sizes[0], rbufoff);
+		bp += off;
+		erroff -= off;
+		rbufoff -= off;
+	}
+
+	for (ii = 1; ii < nents - 1; ii++) {
+		off = genasm_gencache(bp, ii, umem_alloc_sizes[ii], rbufoff);
+		bp += off;
+		rbufoff -= off;
+		erroff -= off;
+	}
+
+	bp += genasm_lastcache(bp, nents - 1, umem_alloc_sizes[nents - 1],
+	    erroff);
+	bp += genasm_frfini(bp, umem_ptc_size, umem_genasm_ofptr);
+	ASSERT(((uintptr_t)bp - total) == (uintptr_t)base);
+
+	return (0);
+}
+
+int
+umem_genasm(int *alloc_sizes, umem_cache_t **caches, int ncaches)
+{
+	int nents, i;
+	uint8_t *mptr;
+	uint8_t *fptr;
+	uint32_t *ptr;
+	uint64_t v, *vptr;
+
+	mptr = (void *)((uintptr_t)&umem_genasm_mptr + 5);
+	fptr = (void *)((uintptr_t)&umem_genasm_fptr + 5);
+	if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
+	    umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
+		return (1);
+
+	/*
+	 * The total number of caches that we can service is the minimum of:
+	 *  o the amount supported by libc
+	 *  o the total number of umem caches
+	 *  o we use a single byte addl, so its 255 / sizeof (uintptr_t). For
+	 *    32-bit, this is 63.
+	 */
+	nents = _tmem_get_nentries();
+
+	if (UMEM_GENASM_MAX32 < nents)
+		nents = UMEM_GENASM_MAX32;
+
+	if (ncaches < nents)
+		nents = ncaches;
+
+	/* Based on our constraints, this is not an error */
+	if (nents == 0 || umem_ptc_size == 0)
+		return (0);
+
+	/* Grab the original malloc and free locations */
+	ptr = (void *)(mptr - 4);
+	umem_genasm_omptr = *ptr + (uintptr_t)mptr;
+	ptr = (void *)(fptr - 4);
+	umem_genasm_ofptr = *ptr + (uintptr_t)fptr;
+
+	/* Take into account the jump */
+	if (genasm_malloc(mptr, umem_genasm_fsize - 5, nents,
+	    alloc_sizes) != 0)
+		return (1);
+
+	if (genasm_free(fptr, umem_genasm_fsize - 5, nents,
+	    alloc_sizes) != 0)
+		return (1);
+
+	/* nop out the jump with a multibyte jump */
+	vptr = (void *)&umem_genasm_mptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+	vptr = (void *)&umem_genasm_fptr;
+	v = MULTINOP;
+	v |= *vptr & (0xffffffULL << 40);
+	(void) atomic_swap_64(vptr, v);
+
+	for (i = 0; i < nents; i++)
+		caches[i]->cache_flags |= UMF_PTC;
+
+	return (0);
+}

init_lib.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)init_lib.c	1.2	05/06/08 SMI" */
@@ -149,7 +148,7 @@ umem_get_max_ncpus(void)
   return info.dwNumberOfProcessors;
 #else
   /* XXX: determine CPU count on other platforms */
-  return (1);
+  #error Cannot detremine CPU count on this platform, please submit a bug (and a patch) for this platform.
 #endif
 
 #endif /* linux */

init_stand.c

@@ -24,7 +24,7 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"@(#)init_stand.c	1.3	05/06/08 SMI"
+/* #pragma ident	"@(#)init_stand.c	1.3	05/06/08 SMI" */
 
 /*
  * Initialization routines for the standalone version of libumem.

linktest_stand.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -25,7 +24,7 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"@(#)linktest_stand.c	1.3	05/06/08 SMI"
+/* #pragma ident	"@(#)linktest_stand.c	1.3	05/06/08 SMI" */
 
 /*
  * This file is used to verify that the standalone's external dependencies
@@ -35,6 +34,7 @@
 void __umem_assert_failed(void) {}
 void _atomic_add_64(void) {}
 void _atomic_add_32_nv(void) {}
+void dladdr1(void) {}
 void bcopy(void) {}
 void bzero(void) {}
 void dladdr1(void) {}
@@ -43,6 +43,7 @@ void getenv(void) {}
 void gethrtime(void) {}
 void membar_producer(void) {}
 void memcpy(void) {}
+void _memcpy(void) {}
 void memset(void) {}
 void snprintf(void) {}
 void strchr(void) {}

malloc.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -25,7 +24,7 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"@(#)malloc.c	1.5	05/06/08 SMI"
+/* #pragma ident	"@(#)malloc.c	1.5	05/06/08 SMI" */
 
 #include "config.h"
 #include <unistd.h>

misc.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)misc.c	1.6	05/06/08 SMI" */

misc.h

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 #ifndef	_MISC_H
@@ -39,6 +38,7 @@
 #endif
 #ifdef HAVE_THREAD_H
 # include <thread.h>
+# include <pthread.h>
 #else
 # include "sol_compat.h"
 #endif

stand_mapfile

@@ -1,13 +1,11 @@
 #
-# Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
-# Use is subject to license terms.
+# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 #
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
-# Common Development and Distribution License, Version 1.0 only
-# (the "License").  You may not use this file except in compliance
-# with the License.
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or http://www.opensolaris.org/os/licensing.

stub_stand.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -23,9 +22,11 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
  */
 
-#pragma ident	"@(#)stub_stand.c	1.3	05/06/08 SMI"
+/* #pragma ident	"@(#)stub_stand.c	1.3	05/06/08 SMI" */
 
 /*
  * Stubs for the standalone to reduce the dependence on external libraries
@@ -124,3 +125,29 @@ issetugid(void)
 {
 	return (1);
 }
+
+int
+_tmem_get_nentries(void)
+{
+	return (0);
+}
+
+uintptr_t
+_tmem_get_base(void)
+{
+	return (0);
+}
+
+/*ARGSUSED*/
+void
+_tmem_set_cleanup(void (*f)(int, void *))
+{
+}
+
+uint64_t
+atomic_swap_64(volatile uint64_t *t, uint64_t v)
+{
+	uint64_t old = *t;
+	*t = v;
+	return (old);
+}

sys/vmem_impl_user.h

@@ -23,7 +23,7 @@
  * Copyright 1999-2002 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 #ifndef _SYS_VMEM_IMPL_USER_H

tmem.c

@@ -0,0 +1,85 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ */
+
+#include "lint.h"
+#include "thr_uberdata.h"
+
+/*
+ * This file implements the private interface with libumem for per-thread
+ * caching umem (ptcumem). For the full details on how tcumem works and how
+ * these functions work, see section 8.4 of the big theory statement in
+ * lib/libumem/common/umem.c.
+ */
+static tmem_func_t tmem_cleanup = NULL;
+
+uintptr_t
+_tmem_get_base(void)
+{
+	return ((uintptr_t)&curthread->ul_tmem - (uintptr_t)curthread);
+}
+
+int
+_tmem_get_nentries(void)
+{
+	return (NTMEMBASE);
+}
+
+void
+_tmem_set_cleanup(tmem_func_t f)
+{
+	tmem_cleanup = f;
+}
+
+/*
+ * This is called by _thrp_exit() to clean up any per-thread allocations that
+ * are still hanging around and haven't been cleaned up.
+ */
+void
+tmem_exit(void)
+{
+	int ii;
+	void *buf, *next;
+	tumem_t *tp = &curthread->ul_tmem;
+
+
+	if (tp->tm_size == 0)
+		return;
+
+	/*
+	 * Since we have something stored here, we need to ensure we declared a
+	 * clean up handler. If we haven't that's broken and our single private
+	 * consumer should be shot.
+	 */
+	if (tmem_cleanup == NULL)
+		abort();
+	for (ii = 0; ii < NTMEMBASE; ii++) {
+		buf = tp->tm_roots[ii];
+		while (buf != NULL) {
+			next = *(void **)buf;
+			tmem_cleanup(buf, ii);
+			buf = next;
+		}
+	}
+}

umem.c

@@ -25,7 +25,7 @@
  * Portions Copyright 2012 Joyent, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident        "@(#)umem.c     1.11    05/06/08 SMI" */
@@ -78,18 +78,22 @@
  *
  * 1. Overview
  * -----------
- * umem is very close to kmem in implementation.  There are four major
+ * umem is very close to kmem in implementation.  There are seven major
  * areas of divergence:
  *
- *      * Initialization
+ *	* Initialization
  *
- *      * CPU handling
+ *	* CPU handling
  *
- *      * umem_update()
+ *	* umem_update()
  *
- *      * KM_SLEEP v.s. UMEM_NOFAIL
+ *	* KM_SLEEP v.s. UMEM_NOFAIL
  *
- *      * lock ordering
+ *	* lock ordering
+ *
+ *	* changing UMEM_MAXBUF
+ *
+ *	* Per-thread caching for malloc/free
  *
  * 2. Initialization
  * -----------------
@@ -97,9 +101,9 @@
  * into it before it is ready.  umem does not have these luxuries. Instead,
  * initialization is divided into two phases:
  *
- *      * library initialization, and
+ *	* library initialization, and
  *
- *      * first use
+ *	* first use
  *
  * umem's full initialization happens at the time of the first allocation
  * request (via malloc() and friends, umem_alloc(), or umem_zalloc()),
@@ -129,13 +133,13 @@
  *
  * There are four different paths from which umem_init() is called:
  *
- *      * from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF,
+ *	* from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF,
  *
- *      * from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF,
+ *	* from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF,
  *
- *      * from umem_cache_create(), and
+ *	* from umem_cache_create(), and
  *
- *      * from memalign(), with align > UMEM_ALIGN.
+ *	* from memalign(), with align > UMEM_ALIGN.
  *
  * The last three just check if umem is initialized, and call umem_init()
  * if it is not.  For performance reasons, the first case is more complicated.
@@ -160,16 +164,16 @@
  * There are a couple race conditions resulting from the initialization
  * code that we have to guard against:
  *
- *      * In umem_cache_create(), there is a special UMC_INTERNAL cflag
- *      that is passed for caches created during initialization.  It
- *      is illegal for a user to try to create a UMC_INTERNAL cache.
- *      This allows initialization to proceed, but any other
- *      umem_cache_create()s will block by calling umem_init().
+ *	* In umem_cache_create(), there is a special UMC_INTERNAL cflag
+ *	that is passed for caches created during initialization.  It
+ *	is illegal for a user to try to create a UMC_INTERNAL cache.
+ *	This allows initialization to proceed, but any other
+ *	umem_cache_create()s will block by calling umem_init().
  *
- *      * Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask
- *      is always zero.  umem_cache_alloc uses cp->cache_cpu_mask to
- *      mask the cpu number.  This prevents a race between grabbing a
- *      cache pointer out of umem_alloc_table and growing the cpu array.
+ *	* Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask
+ *	is always zero.  umem_cache_alloc uses cp->cache_cpu_mask to
+ *	mask the cpu number.  This prevents a race between grabbing a
+ *	cache pointer out of umem_alloc_table and growing the cpu array.
  *
  *
  * 3. CPU handling
@@ -203,16 +207,16 @@
  * -----------------------------------------
  * A given cache is in one of three states:
  *
- * Inactive             cache_uflags is zero, cache_u{next,prev} are NULL
+ * Inactive		cache_uflags is zero, cache_u{next,prev} are NULL
  *
- * Work Requested       cache_uflags is non-zero (but UMU_ACTIVE is not set),
- *                      cache_u{next,prev} link the cache onto the global
- *                      update list
+ * Work Requested	cache_uflags is non-zero (but UMU_ACTIVE is not set),
+ *			cache_u{next,prev} link the cache onto the global
+ *			update list
  *
- * Active               cache_uflags has UMU_ACTIVE set, cache_u{next,prev}
- *                      are NULL, and either umem_update_thr or
- *                      umem_st_update_thr are actively doing work on the
- *                      cache.
+ * Active		cache_uflags has UMU_ACTIVE set, cache_u{next,prev}
+ *			are NULL, and either umem_update_thr or
+ *			umem_st_update_thr are actively doing work on the
+ *			cache.
  *
  * An update can be added to any cache in any state -- if the cache is
  * Inactive, it transitions to being Work Requested.  If the cache is
@@ -249,12 +253,12 @@
  * The update thread spends most of its time in cond_timedwait() on the
  * umem_update_cv.  It wakes up under two conditions:
  *
- *      * The timedwait times out, in which case it needs to run a global
- *      update, or
+ *	* The timedwait times out, in which case it needs to run a global
+ *	update, or
  *
- *      * someone cond_broadcast(3THR)s the umem_update_cv, in which case
- *      it needs to check if there are any caches in the Work Requested
- *      state.
+ *	* someone cond_broadcast(3THR)s the umem_update_cv, in which case
+ *	it needs to check if there are any caches in the Work Requested
+ *	state.
  *
  * When it is time for another global update, umem calls umem_cache_update()
  * on every cache, then calls vmem_update(), which tunes the vmem structures.
@@ -290,19 +294,19 @@
  *
  * Because we locked all of the mutexes, the only possible inconsistancies are:
  *
- *      * a umem_cache_alloc() could leak its buffer.
+ *	* a umem_cache_alloc() could leak its buffer.
  *
- *      * a caller of umem_depot_alloc() could leak a magazine, and all the
- *      buffers contained in it.
+ *	* a caller of umem_depot_alloc() could leak a magazine, and all the
+ *	buffers contained in it.
  *
- *      * a cache could be in the Active update state.  In the child, there
- *      would be no thread actually working on it.
+ *	* a cache could be in the Active update state.  In the child, there
+ *	would be no thread actually working on it.
  *
- *      * a umem_hash_rescale() could leak the new hash table.
+ *	* a umem_hash_rescale() could leak the new hash table.
  *
- *      * a umem_magazine_resize() could be in progress.
+ *	* a umem_magazine_resize() could be in progress.
  *
- *      * a umem_reap() could be in progress.
+ *	* a umem_reap() could be in progress.
  *
  * The memory leaks we can't do anything about.  umem_release_child() resets
  * the update state, moves any caches in the Active state to the Work Requested
@@ -328,24 +332,24 @@
  * that its clients have any particular type of behavior.  Instead,
  * it provides two types of allocations:
  *
- *      * UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on
- *      failure)
+ *	* UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on
+ *	failure)
  *
- *      * UMEM_NOFAIL, which, on failure, calls an optional callback
- *      (registered with umem_nofail_callback()).
+ *	* UMEM_NOFAIL, which, on failure, calls an optional callback
+ *	(registered with umem_nofail_callback()).
  *
  * The callback is invoked with no locks held, and can do an arbitrary
  * amount of work.  It then has a choice between:
  *
- *      * Returning UMEM_CALLBACK_RETRY, which will cause the allocation
- *      to be restarted.
+ *	* Returning UMEM_CALLBACK_RETRY, which will cause the allocation
+ *	to be restarted.
  *
- *      * Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2)
- *      to be invoked with status.  If multiple threads attempt to do
- *      this simultaneously, only one will call exit(2).
+ *	* Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2)
+ *	to be invoked with status.  If multiple threads attempt to do
+ *	this simultaneously, only one will call exit(2).
  *
- *      * Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C),
- *      etc.)
+ *	* Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C),
+ *	etc.)
  *
  * The default callback returns UMEM_CALLBACK_EXIT(255).
  *
@@ -354,16 +358,16 @@
  * close to the original allocation, with no inconsistent state or held
  * locks.  The following steps are taken:
  *
- *      * All invocations of vmem are VM_NOSLEEP.
+ *	* All invocations of vmem are VM_NOSLEEP.
  *
- *      * All constructor callbacks (which can themselves to allocations)
- *      are passed UMEM_DEFAULT as their required allocation argument.  This
- *      way, the constructor will fail, allowing the highest-level allocation
- *      invoke the nofail callback.
+ *	* All constructor callbacks (which can themselves to allocations)
+ *	are passed UMEM_DEFAULT as their required allocation argument.  This
+ *	way, the constructor will fail, allowing the highest-level allocation
+ *	invoke the nofail callback.
  *
- *      If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
- *      the nofail callback does a non-local exit, we will leak the
- *      partially-constructed buffer.
+ *	If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
+ *	the nofail callback does a non-local exit, we will leak the
+ *	partially-constructed buffer.
  *
  *
  * 6. Lock Ordering
@@ -371,26 +375,24 @@
  * umem has a few more locks than kmem does, mostly in the update path.  The
  * overall lock ordering (earlier locks must be acquired first) is:
  *
- *      umem_init_lock
+ *	umem_init_lock
  *
- *      vmem_list_lock
- *      vmem_nosleep_lock.vmpl_mutex
- *      vmem_t's:
- *              vm_lock
- *      sbrk_lock
+ *	vmem_list_lock
+ *	vmem_nosleep_lock.vmpl_mutex
+ *	vmem_t's:
+ *		vm_lock
+ *	sbrk_lock
  *
- *      umem_cache_lock
- *      umem_update_lock
- *      umem_flags_lock
- *      umem_cache_t's:
- *              cache_cpu[*].cc_lock
- *              cache_depot_lock
- *              cache_lock
- *      umem_log_header_t's:
- *              lh_cpu[*].clh_lock
- *              lh_lock
- *
- * \endcode
+ *	umem_cache_lock
+ *	umem_update_lock
+ *	umem_flags_lock
+ *	umem_cache_t's:
+ *		cache_cpu[*].cc_lock
+ *		cache_depot_lock
+ *		cache_lock
+ *	umem_log_header_t's:
+ *		lh_cpu[*].clh_lock
+ *		lh_lock
  *
  * 7. Changing UMEM_MAXBUF
  * -----------------------
@@ -402,6 +404,237 @@
  *
  * The second place to update, which is not required, is the umem_alloc_sizes.
  * These determine the default cache sizes that we're going to support.
+ *
+ * 8. Per-thread caching for malloc/free
+ * -------------------------------------
+ *
+ * "Time is an illusion. Lunchtime doubly so." -- Douglas Adams
+ *
+ * Time may be an illusion, but CPU cycles aren't.  While libumem is designed
+ * to be a highly scalable allocator, that scalability comes with a fixed cycle
+ * penalty even in the absence of contention: libumem must acquire (and release
+ * a per-CPU lock for each allocation.  When contention is low and malloc(3C)
+ * frequency is high, this overhead can dominate execution time.  To alleviate
+ * this, we allow for per-thread caching, a lock-free means of caching recent
+ * deallocations on a per-thread basis for use in satisfying subsequent calls
+ *
+ * In addition to improving performance, we also want to:
+ *	* Minimize fragmentation
+ *	* Not add additional memory overhead (no larger malloc tags)
+ *
+ * In the ulwp_t of each thread there is a private data structure called a
+ * umem_t that looks like:
+ *
+ * typedef struct {
+ * 	size_t	tm_size;
+ * 	void	*tm_roots[NTMEMBASE];  (Currently 16)
+ * } tmem_t;
+ *
+ * Each of the roots is treated as the head of a linked list. Each entry in the
+ * list can be thought of as a void ** which points to the next entry, until one
+ * of them points to NULL. If the head points to NULL, the list is empty.
+ *
+ * Each head corresponds to a umem_cache. Currently there is a linear mapping
+ * where the first root corresponds to the first cache, second root to the
+ * second cache, etc. This works because every allocation that malloc makes to
+ * umem_alloc that can be satisified by a umem_cache will actually return a
+ * number of bytes equal to the size of that cache. Because of this property and
+ * a one to one mapping between caches and roots we can guarantee that every
+ * entry in a given root's list will be able to satisfy the same requests as the
+ * corresponding cache.
+ *
+ * The maximum amount of memory that can be cached in each thread is determined
+ * by the perthread_cache UMEM_OPTION. It corresponds to the umem_ptc_size
+ * value. The default value for this is currently 1 MB. Once umem_init() has
+ * finished this cannot be directly tuned without directly modifying the
+ * instruction text. If, upon calling free(3C), the amount cached would exceed
+ * this maximum, we instead actually return the buffer to the umem_cache instead
+ * of holding onto it in the thread.
+ *
+ * When a thread calls malloc(3C) it first determines which umem_cache it
+ * would be serviced by. If the allocation is not covered by ptcumem it goes to
+ * the normal malloc instead.  Next, it checks if the tmem_root's list is empty
+ * or not. If it is empty, we instead go and allocate the memory from
+ * umem_alloc. If it is not empty, we remove the head of the list, set the
+ * appropriate malloc tags, and return that buffer.
+ *
+ * When a thread calls free(3C) it first looks at the malloc tag and if it is
+ * invalid or the allocation exceeds the largest cache in ptcumem and sends it
+ * off to the original free() to handle and clean up appropriately. Next, it
+ * checks if the allocation size is covered by one of the per-thread roots and
+ * if it isn't, it passes it off to the original free() to be released. Finally,
+ * before it inserts this buffer as the head, it checks if adding this buffer
+ * would put the thread over its maximum cache size. If it would, it frees the
+ * buffer back to the umem_cache. Otherwise it increments the threads total
+ * cached amount and makes the buffer the new head of the appropriate tm_root.
+ *
+ * When a thread exits, all of the buffers that it has in its per-thread cache
+ * will be passed to umem_free() and returned to the appropriate umem_cache.
+ *
+ * 8.1 Handling addition and removal of umem_caches
+ * ------------------------------------------------
+ *
+ * The set of umem_caches that are used to back calls to umem_alloc() and
+ * ultimately malloc() are determined at program execution time. The default set
+ * of caches is defined below in umem_alloc_sizes[]. Various umem_options exist
+ * that modify the set of caches: size_add, size_clear, and size_remove. Because
+ * the set of caches can only be determined once umem_init() has been called and
+ * we have the additional goals of minimizing additional fragmentation and
+ * metadata space overhead in the malloc tags, this forces our hand to go down a
+ * slightly different path: the one tread by fasttrap and trapstat.
+ *
+ * During umem_init we're going to dynamically construct a new version of
+ * malloc(3C) and free(3C) that utilizes the known cache sizes and then ensure
+ * that ptcmalloc and ptcfree replace malloc and free as entries in the plt. If
+ * ptcmalloc and ptcfree cannot handle a request, they simply jump to the
+ * original libumem implementations.
+ *
+ * After creating all of the umem_caches, but before making them visible,
+ * umem_cache_init checks that umem_genasm_supported is non-zero. This value is
+ * set by each architecture in $ARCH/umem_genasm.c to indicate whether or not
+ * they support this. If the value is zero, then this process is skipped.
+ * Similarly, if the cache size has been tuned to zero by UMEM_OPTIONS, then
+ * this is also skipped.
+ *
+ * In umem_genasm.c, each architecture's implementation implements a single
+ * function called umem_genasm() that is responsible for generating the
+ * appropriate versions of ptcmalloc() and ptcfree(), placing them in the
+ * appropriate memory location, and finally doing the switch from malloc() and
+ * free() to ptcmalloc() and ptcfree().  Once the change has been made, there is
+ * no way to switch back, short of restarting the program or modifying program
+ * text with mdb.
+ *
+ * 8.2 Modifying the Procedure Linkage Table (PLT)
+ * -----------------------------------------------
+ *
+ * The last piece of this puzzle is how we actually jam ptcmalloc() into the
+ * PLT.  The dyanmic linker has support for global and local audit libraries.
+ * For the full explanation of audit libraries consult the Linkers and Libraries
+ * guide or the linker source. A local auditer can attach to a single library
+ * and interpose on all of the relocations that come in from and leave to that
+ * same library. To facilitate our work, we have created a local audit library
+ * for libumem that is called libumem_trampoline and is located in
+ * lib/libumem_trampoline/.
+ *
+ * When any resolution is done to malloc(), the audit library allows us to
+ * replace the address with an address that it specifies. There are two 4k
+ * sections in the libumem_trampoline's bss which we use as the stomping grounds
+ * for ptcmalloc and ptcfree. When the audit library audits the malloc and free
+ * functions from libumem, it encodes their address and sets its buffers to
+ * contain a simple trampoline which consists of a jmp instruction and a four
+ * byte offset to the original malloc and free. libumem_trampoline's mapfile
+ * explicitly makes its bss rwx instead of rw to support this.
+ *
+ * When umem_genasm() is called, it uses a similar mechanism to get the address
+ * and size of the trampoline libraries malloc (mbuf) and free (fbuf) buffers.
+ * After validating that the size will be able to contain all of the
+ * instructions, it starts laying out ptcmalloc and ptcfree at mbuf[4] and
+ * fbuf[4]. Once both have been successfully generated, umem_genasm() stores a
+ * single five byte nop over the original jump.
+ *
+ * 8.3 umem_genasm()
+ * -----------------
+ *
+ * umem_genasm() is currently implemented for i386 and amd64. This section
+ * describes the theory behind the construction. For specific byte code to
+ * assembly instructions and niceish C and asm versions of ptcmalloc and
+ * ptcfree, see the individual umem_genasm.c files. The layout consists of the
+ * following sections:
+ *
+ *	o. function-specfic prologue
+ *	o. function-generic cache-selecting elements
+ *	o. function-specific epilogue
+ *
+ * There are three different generic cache elements that exist:
+ *
+ *	o. the last or only cache
+ *	o. the intermediary caches if more than two
+ *	o. the first one if more than one cache
+ *
+ * The malloc and free prologues and epilogues mimic the necessary portions of
+ * libumem's malloc and free. This includes things like checking for size
+ * overflow, setting and verifying the malloc tags.
+ *
+ * It is an important constraint that these functions do not make use of the
+ * call instruction. The only jmp outside of the individual functions is to the
+ * original libumem malloc and free respectively. Because doing things like
+ * setting errno or raising an internal umem error on improper malloc tags would
+ * require using calls into the PLT, whenever we encounter one of those cases we
+ * just jump to the original malloc and free functions reusing the same stack
+ * frame.
+ *
+ * Each of the above sections, the three caches, and the malloc and free
+ * prologue and epilogue are implemented as blocks of machine code with the
+ * corresponding assembly in comments. There are known offsets into each block
+ * that corresponds to locations of data and addresses that we only know at run
+ * time. These blocks are copied as necessary and the blanks filled in
+ * appropriately.
+ *
+ * As mentioned in section 8.2, the trampoline library uses specifically named
+ * variables to communicate the buffers and size to use. These variables are:
+ *
+ *	o. umem_genasm_mptr: The buffer for ptcmalloc
+ *	o. umem_genasm_msize: The size in bytes of the above buffer
+ *	o. umem_genasm_fptr: The buffer for ptcfree
+ *	o. umem_genasm_fsize: The size in bytes of the above buffer
+ *
+ * Finally, to enable the generated assembly we need to remove the previous jump
+ * to the actual malloc that exists at the start of these buffers. This is a
+ * five byte region. We could zero out the jump offset to be a jmp +0, but
+ * using nops can be faster. We specifically use a single five byte nop which is
+ * faster. The opcode for the five byte nop is 0x 0f 1f 44 00 00. On x86,
+ * remember integers are little endian, so it will be written the other way
+ * around.
+ *
+ * 8.4 Interface with libc.so
+ * --------------------------
+ *
+ * The tmem_t structure as described in the beginning of section 8, is part of a
+ * private interface with libc. There are three functions that exist to cover
+ * this. They are not documented in man pages or header files. They are in the
+ * SUNWprivate part of libc's makefile.
+ *
+ *	o. _tmem_get_base(void)
+ *
+ * 	Returns the offset from the ulwp_t (curthread) to the tmem_t structure.
+ * 	This is a constant for all threads and is effectively a way to to do
+ * 	::offsetof ulwp_t ul_tmem without having to know the specifics of the
+ * 	structure outside of libc.
+ *
+ *	o. _tmem_get_nentries(void)
+ *
+ *	Returns the number of roots that exist in the tmem_t. This is one part
+ *	of the cap on the number of umem_caches that we can back with tmem.
+ *
+ *	o. _tmem_set_cleanup(void (*)(void *, int))
+ *
+ *	This sets a clean up handler that gets called back when a thread exits.
+ *	There is one call per buffer, the void * is a pointer to the buffer on
+ *	the list, the int is the index into the roots array for this buffer.
+ *
+ * 8.5 Tuning and disabling per-thread caching
+ * -------------------------------------------
+ *
+ * There is only one tunable for per-thread caching:  the amount of memory each
+ * thread should be able to cache.  This is specified via the perthread_cache
+ * UMEM_OPTION option.  No attempt is made to to sanity check the specified
+ * value; the limit is simply the maximum value of a size_t.
+ *
+ * If the perthread_cache UMEM_OPTION is set to zero, nomagazines was requested,
+ * or UMEM_DEBUG has been turned on then we will never call into umem_genasm;
+ * however, the trampoline audit library and jump will still be in place.
+ *
+ * 8.6 Observing efficacy of per-thread caching
+ * --------------------------------------------
+ *
+ * To understand the efficacy of per-thread caching, use the ::umastat dcmd
+ * to see the percentage of capacity consumed on a per-thread basis, the
+ * degree to which each umem cache contributes to per-thread cache consumption,
+ * and the number of buffers in per-thread caches on a per-umem cache basis.
+ * If more detail is required, the specific buffers in a per-thread cache can
+ * be iterated over with the umem_ptc_* walkers. (These walkers allow an
+ * optional ulwp_t to be specified to iterate only over a particular thread's
+ * cache.)
  */
 
 #include "config.h"
@@ -524,8 +757,10 @@ size_t umem_lite_minsize = 0;   /* minimum buffer size for UMF_LITE */
 size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */
 size_t umem_maxverify;          /* maximum bytes to inspect in debug routines */
 size_t umem_minfirewall;        /* hardware-enforced redzone threshold */
+size_t umem_ptc_size = 1048576;	/* size of per-thread cache (in bytes) */
 
 uint_t umem_flags = 0;
+uintptr_t umem_tmem_off;
 
 mutex_t                 umem_init_lock = DEFAULTMUTEX;          /* locks initialization */
 cond_t                  umem_init_cv = DEFAULTCV;               /* initialization CV */
@@ -533,6 +768,8 @@ thread_t                umem_init_thr;          /* thread initializing */
 int                     umem_init_env_ready;    /* environ pre-initted */
 int                     umem_ready = UMEM_READY_STARTUP;
 
+int			umem_ptc_enabled;	/* per-thread caching enabled */
+
 static umem_nofail_callback_t *nofail_callback;
 static mutex_t          umem_nofail_exit_lock = DEFAULTMUTEX;
 static thread_t         umem_nofail_exit_thr;
@@ -2917,6 +3154,24 @@ umem_alloc_sizes_remove(size_t size)
         umem_alloc_sizes[i] = 0;
 }
 
+/*
+ * We've been called back from libc to indicate that thread is terminating and
+ * that it needs to release the per-thread memory that it has. We get to know
+ * which entry in the thread's tmem array the allocation came from. Currently
+ * this refers to first n umem_caches which makes this a pretty simple indexing
+ * job.
+ */
+static void
+umem_cache_tmem_cleanup(void *buf, int entry)
+{
+	size_t size;
+	umem_cache_t *cp;
+
+	size = umem_alloc_sizes[entry];
+	cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT];
+	_umem_cache_free(cp, buf);
+}
+
 static int
 umem_cache_init(void)
 {
@@ -3032,6 +3287,16 @@ umem_cache_init(void)
                 umem_alloc_caches[i] = cp;
         }
 
+	umem_tmem_off = _tmem_get_base();
+	_tmem_set_cleanup(umem_cache_tmem_cleanup);
+
+	if (umem_genasm_supported && !(umem_flags & UMF_DEBUG) &&
+	    !(umem_flags & UMF_NOMAGAZINE) &&
+	    umem_ptc_size > 0) {
+		umem_ptc_enabled = umem_genasm(umem_alloc_sizes,
+		    umem_alloc_caches, i) == 0 ? 1 : 0;
+	}
+
         /*
          * Initialization cannot fail at this point.  Make the caches
          * visible to umem_alloc() and friends.

umem_agent_support.c

@@ -23,7 +23,7 @@
  * Copyright 2002 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)umem_agent_support.c	1.2	05/06/08 SMI" */

umem_base.h

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -22,6 +21,8 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_UMEM_BASE_H
@@ -76,6 +77,8 @@ extern volatile uint32_t umem_reaping;
 #define	UMEM_REAP_ADDING	0x00000001	/* umem_reap() is active */
 #define	UMEM_REAP_ACTIVE	0x00000002	/* update thread is reaping */
 
+extern uintptr_t umem_tmem_off;
+
 /*
  * umem.c: tunables
  */
@@ -98,6 +101,7 @@ extern size_t umem_lite_minsize;
 extern size_t umem_lite_maxalign;
 extern size_t umem_maxverify;
 extern size_t umem_minfirewall;
+extern size_t umem_ptc_size;
 
 extern uint32_t umem_flags;
 
@@ -140,6 +144,12 @@ extern int umem_create_update_thread(void);
 void umem_setup_envvars(int);
 void umem_process_envvars(void);
 
+/*
+ * umem_genasm.c: private interfaces
+ */
+extern int umem_genasm_supported;
+extern int umem_genasm(int *, umem_cache_t **, int);
+
 #ifdef	__cplusplus
 }
 #endif

umem_fail.c

@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -24,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)umem_fail.c	1.4	05/06/08 SMI" */
@@ -166,5 +165,4 @@ __umem_assert_failed(const char *assertion, const char *file, int line)
 	umem_panic("Assertion failed: %s, file %s, line %d\n",
 	    assertion, file, line);
 	/*NOTREACHED*/
-	return (0);
 }

umem_impl.h

@@ -21,11 +21,11 @@
  */
 /*
  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
- *
- * Portions Copyright 2012 Joyent, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc.  All rights reserved.
+ *
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 #ifndef _UMEM_IMPL_H
@@ -78,6 +78,7 @@ extern "C" {
 
 #define	UMF_HASH	0x00000200	/* cache has hash table */
 #define	UMF_RANDOMIZE	0x00000400	/* randomize other umem_flags */
+#define	UMF_PTC		0x00000800	/* cache has per-thread caching */
 
 #define	UMF_BUFTAG	(UMF_DEADBEEF | UMF_REDZONE)
 #define	UMF_TOUCH	(UMF_BUFTAG | UMF_LITE | UMF_CONTENTS)
@@ -418,6 +419,13 @@ extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
 extern int umem_add(caddr_t, size_t);
 #endif
 
+/*
+ * Private interface with libc for tcumem.
+ */
+extern uintptr_t _tmem_get_base(void);
+extern int _tmem_get_nentries(void);
+extern void _tmem_set_cleanup(void(*)(void *, int));
+
 #ifdef	__cplusplus
 }
 #endif

umem_test.c

@@ -3,11 +3,13 @@
 #include <string.h>
 
 #include "umem.h"
+#define UMEM_STANDALONE
+#include "umem_impl.h"
 
 int main(int argc, char *argv[])
 {
   char *foo;
-  
+
   umem_startup(NULL, 0, 0, NULL, NULL);
 
   foo = umem_alloc(32, UMEM_DEFAULT);
@@ -20,4 +22,3 @@ int main(int argc, char *argv[])
 
   return EXIT_SUCCESS;
 }
-

umem_update_thread.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)umem_update_thread.c	1.2	05/06/08 SMI" */

umemdbg/mdb/common/libumem.c

@@ -0,0 +1,610 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ */
+
+#include "umem.h"
+#include <libproc.h>
+#include <mdb/mdb_modapi.h>
+
+#include "kgrep.h"
+#include "leaky.h"
+#include "misc.h"
+#include "proc_kludges.h"
+
+#include <umem_impl.h>
+#include <sys/vmem_impl_user.h>
+#include <thr_uberdata.h>
+#include <stdio.h>
+
+#include "umem_pagesize.h"
+
+typedef struct datafmt {
+	char	*hdr1;
+	char	*hdr2;
+	char	*dashes;
+	char	*fmt;
+} datafmt_t;
+
+static datafmt_t ptcfmt[] = {
+	{ "   ",	"tid",		"---",		"%3u "		},
+	{ " memory",	" cached",	"-------",	"%7lH "		},
+	{ "  %",	"cap",		"---",		"%3u "		},
+	{ "  %",	NULL,		"---",		"%3u "		},
+	{ NULL,		NULL,		NULL,		NULL		}
+};
+
+static datafmt_t umemfmt[] = {
+	{ "cache                    ", "name                     ",
+	"-------------------------", "%-25s "				},
+	{ "   buf",	"  size",	"------",	"%6u "		},
+	{ "    buf",	" in use",	"-------",	"%7u "		},
+	{ "    buf",	" in ptc",	"-------",	"%7s "		},
+	{ "    buf",	"  total",	"-------",	"%7u "		},
+	{ " memory",	" in use",	"-------",	"%7H "		},
+	{ "    alloc",	"  succeed",	"---------",	"%9u "		},
+	{ "alloc",	" fail",	"-----",	"%5llu"		},
+	{ NULL,		NULL,		NULL,		NULL		}
+};
+
+static datafmt_t vmemfmt[] = {
+	{ "vmem                     ", "name                     ",
+	"-------------------------", "%-*s "				},
+	{ "   memory",	"   in use",	"---------",	"%9H "		},
+	{ "    memory",	"     total",	"----------",	"%10H "		},
+	{ "   memory",	"   import",	"---------",	"%9H "		},
+	{ "    alloc",	"  succeed",	"---------",	"%9llu "	},
+	{ "alloc",	" fail",	"-----",	"%5llu "	},
+	{ NULL,		NULL,		NULL,		NULL		}
+};
+
+/*ARGSUSED*/
+static int
+umastat_cpu_avail(uintptr_t addr, const umem_cpu_cache_t *ccp, int *avail)
+{
+	if (ccp->cc_rounds > 0)
+		*avail += ccp->cc_rounds;
+	if (ccp->cc_prounds > 0)
+		*avail += ccp->cc_prounds;
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_cpu_alloc(uintptr_t addr, const umem_cpu_cache_t *ccp, int *alloc)
+{
+	*alloc += ccp->cc_alloc;
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_slab_avail(uintptr_t addr, const umem_slab_t *sp, int *avail)
+{
+	*avail += sp->slab_chunks - sp->slab_refcnt;
+
+	return (WALK_NEXT);
+}
+
+typedef struct umastat_vmem {
+	uintptr_t kv_addr;
+	struct umastat_vmem *kv_next;
+	int kv_meminuse;
+	int kv_alloc;
+	int kv_fail;
+} umastat_vmem_t;
+
+/*ARGSUSED*/
+static int
+umastat_cache_nptc(uintptr_t addr, const umem_cache_t *cp, int *nptc)
+{
+	if (!(cp->cache_flags & UMF_PTC))
+		return (WALK_NEXT);
+
+	(*nptc)++;
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_cache_hdr(uintptr_t addr, const umem_cache_t *cp, void *ignored)
+{
+	if (!(cp->cache_flags & UMF_PTC))
+		return (WALK_NEXT);
+
+	mdb_printf("%3d ", cp->cache_bufsize);
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_lwp_ptc(uintptr_t addr, void *buf, int *nbufs)
+{
+	(*nbufs)++;
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_lwp_cache(uintptr_t addr, const umem_cache_t *cp, ulwp_t *ulwp)
+{
+	char walk[60];
+	int nbufs = 0;
+
+	if (!(cp->cache_flags & UMF_PTC))
+		return (WALK_NEXT);
+
+	(void) snprintf(walk, sizeof (walk), "umem_ptc_%d", cp->cache_bufsize);
+
+	if (mdb_pwalk(walk, (mdb_walk_cb_t)umastat_lwp_ptc,
+	    &nbufs, (uintptr_t)ulwp->ul_self) == -1) {
+		mdb_warn("unable to walk '%s'", walk);
+		return (WALK_ERR);
+	}
+
+	mdb_printf("%3d ", ulwp->ul_tmem.tm_size ?
+	    (nbufs * cp->cache_bufsize * 100) / ulwp->ul_tmem.tm_size : 0);
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_lwp(uintptr_t addr, const ulwp_t *ulwp, void *ignored)
+{
+	size_t size;
+	datafmt_t *dfp = ptcfmt;
+
+	mdb_printf((dfp++)->fmt, ulwp->ul_lwpid);
+	mdb_printf((dfp++)->fmt, ulwp->ul_tmem.tm_size);
+
+	if (umem_readvar(&size, "umem_ptc_size") == -1) {
+		mdb_warn("unable to read 'umem_ptc_size'");
+		return (WALK_ERR);
+	}
+
+	mdb_printf((dfp++)->fmt, (ulwp->ul_tmem.tm_size * 100) / size);
+
+	if (mdb_walk("umem_cache",
+	    (mdb_walk_cb_t)umastat_lwp_cache, (void *)ulwp) == -1) {
+		mdb_warn("can't walk 'umem_cache'");
+		return (WALK_ERR);
+	}
+
+	mdb_printf("\n");
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_cache_ptc(uintptr_t addr, const void *ignored, int *nptc)
+{
+	(*nptc)++;
+	return (WALK_NEXT);
+}
+
+static int
+umastat_cache(uintptr_t addr, const umem_cache_t *cp, umastat_vmem_t **kvp)
+{
+	umastat_vmem_t *kv;
+	datafmt_t *dfp = umemfmt;
+	char buf[10];
+	int magsize;
+
+	int avail, alloc, total, nptc = 0;
+	size_t meminuse = (cp->cache_slab_create - cp->cache_slab_destroy) *
+	    cp->cache_slabsize;
+
+	mdb_walk_cb_t cpu_avail = (mdb_walk_cb_t)umastat_cpu_avail;
+	mdb_walk_cb_t cpu_alloc = (mdb_walk_cb_t)umastat_cpu_alloc;
+	mdb_walk_cb_t slab_avail = (mdb_walk_cb_t)umastat_slab_avail;
+
+	magsize = umem_get_magsize(cp);
+
+	alloc = cp->cache_slab_alloc + cp->cache_full.ml_alloc;
+	avail = cp->cache_full.ml_total * magsize;
+	total = cp->cache_buftotal;
+
+	(void) mdb_pwalk("umem_cpu_cache", cpu_alloc, &alloc, addr);
+	(void) mdb_pwalk("umem_cpu_cache", cpu_avail, &avail, addr);
+	(void) mdb_pwalk("umem_slab_partial", slab_avail, &avail, addr);
+
+	if (cp->cache_flags & UMF_PTC) {
+		char walk[60];
+
+		(void) snprintf(walk, sizeof (walk),
+		    "umem_ptc_%d", cp->cache_bufsize);
+
+		if (mdb_walk(walk,
+		    (mdb_walk_cb_t)umastat_cache_ptc, &nptc) == -1) {
+			mdb_warn("unable to walk '%s'", walk);
+			return (WALK_ERR);
+		}
+
+		(void) snprintf(buf, sizeof (buf), "%d", nptc);
+	}
+
+	for (kv = *kvp; kv != NULL; kv = kv->kv_next) {
+		if (kv->kv_addr == (uintptr_t)cp->cache_arena)
+			goto out;
+	}
+
+	kv = mdb_zalloc(sizeof (umastat_vmem_t), UM_SLEEP | UM_GC);
+	kv->kv_next = *kvp;
+	kv->kv_addr = (uintptr_t)cp->cache_arena;
+	*kvp = kv;
+out:
+	kv->kv_meminuse += meminuse;
+	kv->kv_alloc += alloc;
+	kv->kv_fail += cp->cache_alloc_fail;
+
+	mdb_printf((dfp++)->fmt, cp->cache_name);
+	mdb_printf((dfp++)->fmt, cp->cache_bufsize);
+	mdb_printf((dfp++)->fmt, total - avail);
+	mdb_printf((dfp++)->fmt, cp->cache_flags & UMF_PTC ? buf : "-");
+	mdb_printf((dfp++)->fmt, total);
+	mdb_printf((dfp++)->fmt, meminuse);
+	mdb_printf((dfp++)->fmt, alloc);
+	mdb_printf((dfp++)->fmt, cp->cache_alloc_fail);
+	mdb_printf("\n");
+
+	return (WALK_NEXT);
+}
+
+static int
+umastat_vmem_totals(uintptr_t addr, const vmem_t *v, umastat_vmem_t *kv)
+{
+	while (kv != NULL && kv->kv_addr != addr)
+		kv = kv->kv_next;
+
+	if (kv == NULL || kv->kv_alloc == 0)
+		return (WALK_NEXT);
+
+	mdb_printf("Total [%s]%*s %6s %7s %7s %7s %7H %9u %5u\n", v->vm_name,
+	    17 - strlen(v->vm_name), "", "", "", "", "",
+	    kv->kv_meminuse, kv->kv_alloc, kv->kv_fail);
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+static int
+umastat_vmem(uintptr_t addr, const vmem_t *v, void *ignored)
+{
+	datafmt_t *dfp = vmemfmt;
+	uintptr_t paddr;
+	vmem_t parent;
+	int ident = 0;
+
+	for (paddr = (uintptr_t)v->vm_source; paddr != NULL; ident += 4) {
+		if (mdb_vread(&parent, sizeof (parent), paddr) == -1) {
+			mdb_warn("couldn't trace %p's ancestry", addr);
+			ident = 0;
+			break;
+		}
+		paddr = (uintptr_t)parent.vm_source;
+	}
+
+	mdb_printf("%*s", ident, "");
+	mdb_printf((dfp++)->fmt, 25 - ident, v->vm_name);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_mem_inuse);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_mem_total);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_mem_import);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_alloc);
+	mdb_printf((dfp++)->fmt, v->vm_kstat.vk_fail);
+
+	mdb_printf("\n");
+
+	return (WALK_NEXT);
+}
+
+/*ARGSUSED*/
+int
+umastat(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
+{
+	umastat_vmem_t *kv = NULL;
+	datafmt_t *dfp;
+	int nptc = 0, i;
+
+	if (argc != 0)
+		return (DCMD_USAGE);
+
+	/*
+	 * We need to determine if we have any caches that have per-thread
+	 * caching enabled.
+	 */
+	if (mdb_walk("umem_cache",
+	    (mdb_walk_cb_t)umastat_cache_nptc, &nptc) == -1) {
+		mdb_warn("can't walk 'umem_cache'");
+		return (DCMD_ERR);
+	}
+
+	if (nptc) {
+		for (dfp = ptcfmt; dfp->hdr2 != NULL; dfp++)
+			mdb_printf("%s ", dfp->hdr1);
+
+		for (i = 0; i < nptc; i++)
+			mdb_printf("%s ", dfp->hdr1);
+
+		mdb_printf("\n");
+
+		for (dfp = ptcfmt; dfp->hdr2 != NULL; dfp++)
+			mdb_printf("%s ", dfp->hdr2);
+
+		if (mdb_walk("umem_cache",
+		    (mdb_walk_cb_t)umastat_cache_hdr, NULL) == -1) {
+			mdb_warn("can't walk 'umem_cache'");
+			return (DCMD_ERR);
+		}
+
+		mdb_printf("\n");
+
+		for (dfp = ptcfmt; dfp->hdr2 != NULL; dfp++)
+			mdb_printf("%s ", dfp->dashes);
+
+		for (i = 0; i < nptc; i++)
+			mdb_printf("%s ", dfp->dashes);
+
+		mdb_printf("\n");
+
+		if (mdb_walk("ulwp", (mdb_walk_cb_t)umastat_lwp, NULL) == -1) {
+			mdb_warn("can't walk 'ulwp'");
+			return (DCMD_ERR);
+		}
+
+		mdb_printf("\n");
+	}
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->hdr1);
+	mdb_printf("\n");
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->hdr2);
+	mdb_printf("\n");
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->dashes);
+	mdb_printf("\n");
+
+	if (mdb_walk("umem_cache", (mdb_walk_cb_t)umastat_cache, &kv) == -1) {
+		mdb_warn("can't walk 'umem_cache'");
+		return (DCMD_ERR);
+	}
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s%s", dfp == umemfmt ? "" : " ", dfp->dashes);
+	mdb_printf("\n");
+
+	if (mdb_walk("vmem", (mdb_walk_cb_t)umastat_vmem_totals, kv) == -1) {
+		mdb_warn("can't walk 'vmem'");
+		return (DCMD_ERR);
+	}
+
+	for (dfp = umemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->dashes);
+	mdb_printf("\n");
+
+	mdb_printf("\n");
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->hdr1);
+	mdb_printf("\n");
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->hdr2);
+	mdb_printf("\n");
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->dashes);
+	mdb_printf("\n");
+
+	if (mdb_walk("vmem", (mdb_walk_cb_t)umastat_vmem, NULL) == -1) {
+		mdb_warn("can't walk 'vmem'");
+		return (DCMD_ERR);
+	}
+
+	for (dfp = vmemfmt; dfp->hdr1 != NULL; dfp++)
+		mdb_printf("%s ", dfp->dashes);
+	mdb_printf("\n");
+	return (DCMD_OK);
+}
+
+/*
+ * kmdb doesn't use libproc, and thus doesn't have any prmap_t's to walk.
+ * We have other ways to grep kmdb's address range.
+ */
+#ifndef _KMDB
+
+typedef struct ugrep_walk_data {
+	kgrep_cb_func *ug_cb;
+	void *ug_cbdata;
+} ugrep_walk_data_t;
+
+/*ARGSUSED*/
+int
+ugrep_mapping_cb(uintptr_t addr, const void *prm_arg, void *data)
+{
+	ugrep_walk_data_t *ug = data;
+	const prmap_t *prm = prm_arg;
+
+	return (ug->ug_cb(prm->pr_vaddr, prm->pr_vaddr + prm->pr_size,
+	    ug->ug_cbdata));
+}
+
+int
+kgrep_subr(kgrep_cb_func *cb, void *cbdata)
+{
+	ugrep_walk_data_t ug;
+
+	prockludge_add_walkers();
+
+	ug.ug_cb = cb;
+	ug.ug_cbdata = cbdata;
+
+	if (mdb_walk(KLUDGE_MAPWALK_NAME, ugrep_mapping_cb, &ug) == -1) {
+		mdb_warn("Unable to walk "KLUDGE_MAPWALK_NAME);
+		return (DCMD_ERR);
+	}
+
+	prockludge_remove_walkers();
+	return (DCMD_OK);
+}
+
+size_t
+kgrep_subr_pagesize(void)
+{
+	return (PAGESIZE);
+}
+
+#endif /* !_KMDB */
+
+static const mdb_dcmd_t dcmds[] = {
+
+	/* from libumem.c */
+	{ "umastat", NULL, "umem allocator stats", umastat },
+
+	/* from misc.c */
+	{ "umem_debug", NULL, "toggle umem dcmd/walk debugging", umem_debug},
+
+	/* from umem.c */
+	{ "umem_status", NULL, "Print umem status and message buffer",
+		umem_status },
+	{ "allocdby", ":", "given a thread, print its allocated buffers",
+		allocdby },
+	{ "bufctl", ":[-vh] [-a addr] [-c caller] [-e earliest] [-l latest] "
+		"[-t thd]", "print or filter a bufctl", bufctl, bufctl_help },
+	{ "bufctl_audit", ":", "print a bufctl_audit", bufctl_audit },
+	{ "freedby", ":", "given a thread, print its freed buffers", freedby },
+	{ "umalog", "[ fail | slab ]",
+	    "display umem transaction log and stack traces", umalog },
+	{ "umausers", "[-ef] [cache ...]", "display current medium and large "
+		"users of the umem allocator", umausers },
+	{ "umem_cache", "?", "print a umem cache", umem_cache },
+	{ "umem_log", "?", "dump umem transaction log", umem_log },
+	{ "umem_malloc_dist", "[-dg] [-b maxbins] [-B minbinsize]",
+		"report distribution of outstanding malloc()s",
+		umem_malloc_dist, umem_malloc_dist_help },
+	{ "umem_malloc_info", "?[-dg] [-b maxbins] [-B minbinsize]",
+		"report information about malloc()s by cache",
+		umem_malloc_info, umem_malloc_info_help },
+	{ "umem_verify", "?", "check integrity of umem-managed memory",
+		umem_verify },
+	{ "vmem", "?", "print a vmem_t", vmem },
+	{ "vmem_seg", ":[-sv] [-c caller] [-e earliest] [-l latest] "
+		"[-m minsize] [-M maxsize] [-t thread] [-T type]",
+		"print or filter a vmem_seg", vmem_seg, vmem_seg_help },
+
+#ifndef _KMDB
+	/* from ../genunix/kgrep.c + libumem.c */
+	{ "ugrep", KGREP_USAGE, "search user address space for a pointer",
+	    kgrep, kgrep_help },
+
+	/* from ../genunix/leaky.c + leaky_subr.c */
+	{ "findleaks", FINDLEAKS_USAGE, "search for potential memory leaks",
+	    findleaks, findleaks_help },
+#endif
+
+	{ NULL }
+};
+
+static const mdb_walker_t walkers[] = {
+
+	/* from umem.c */
+	{ "allocdby", "given a thread, walk its allocated bufctls",
+		allocdby_walk_init, allocdby_walk_step, allocdby_walk_fini },
+	{ "bufctl", "walk a umem cache's bufctls",
+		bufctl_walk_init, umem_walk_step, umem_walk_fini },
+	{ "bufctl_history", "walk the available history of a bufctl",
+		bufctl_history_walk_init, bufctl_history_walk_step,
+		bufctl_history_walk_fini },
+	{ "freectl", "walk a umem cache's free bufctls",
+		freectl_walk_init, umem_walk_step, umem_walk_fini },
+	{ "freedby", "given a thread, walk its freed bufctls",
+		freedby_walk_init, allocdby_walk_step, allocdby_walk_fini },
+	{ "freemem", "walk a umem cache's free memory",
+		freemem_walk_init, umem_walk_step, umem_walk_fini },
+	{ "umem", "walk a umem cache",
+		umem_walk_init, umem_walk_step, umem_walk_fini },
+	{ "umem_cpu", "walk the umem CPU structures",
+		umem_cpu_walk_init, umem_cpu_walk_step, umem_cpu_walk_fini },
+	{ "umem_cpu_cache", "given a umem cache, walk its per-CPU caches",
+		umem_cpu_cache_walk_init, umem_cpu_cache_walk_step, NULL },
+	{ "umem_hash", "given a umem cache, walk its allocated hash table",
+		umem_hash_walk_init, umem_hash_walk_step, umem_hash_walk_fini },
+	{ "umem_log", "walk the umem transaction log",
+		umem_log_walk_init, umem_log_walk_step, umem_log_walk_fini },
+	{ "umem_slab", "given a umem cache, walk its slabs",
+		umem_slab_walk_init, umem_slab_walk_step, NULL },
+	{ "umem_slab_partial",
+	    "given a umem cache, walk its partially allocated slabs (min 1)",
+		umem_slab_walk_partial_init, umem_slab_walk_step, NULL },
+	{ "vmem", "walk vmem structures in pre-fix, depth-first order",
+		vmem_walk_init, vmem_walk_step, vmem_walk_fini },
+	{ "vmem_alloc", "given a vmem_t, walk its allocated vmem_segs",
+		vmem_alloc_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+	{ "vmem_free", "given a vmem_t, walk its free vmem_segs",
+		vmem_free_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+	{ "vmem_postfix", "walk vmem structures in post-fix, depth-first order",
+		vmem_walk_init, vmem_postfix_walk_step, vmem_walk_fini },
+	{ "vmem_seg", "given a vmem_t, walk all of its vmem_segs",
+		vmem_seg_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+	{ "vmem_span", "given a vmem_t, walk its spanning vmem_segs",
+		vmem_span_walk_init, vmem_seg_walk_step, vmem_seg_walk_fini },
+
+#ifndef _KMDB
+	/* from ../genunix/leaky.c + leaky_subr.c */
+	{ "leak", "given a leak ctl, walk other leaks w/ that stacktrace",
+		leaky_walk_init, leaky_walk_step, leaky_walk_fini },
+	{ "leakbuf", "given a leak ctl, walk addr of leaks w/ that stacktrace",
+		leaky_walk_init, leaky_buf_walk_step, leaky_walk_fini },
+#endif
+
+	{ NULL }
+};
+
+static const mdb_modinfo_t modinfo = {MDB_API_VERSION, dcmds, walkers};
+
+const mdb_modinfo_t *
+_mdb_init(void)
+{
+	if (umem_init() != 0)
+		return (NULL);
+
+	return (&modinfo);
+}
+
+void
+_mdb_fini(void)
+{
+#ifndef _KMDB
+	leaky_cleanup(1);
+#endif
+}

umemdbg/mdb/common/misc.c

@@ -0,0 +1,110 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include "misc.h"
+
+#define	UMEM_OBJNAME "libumem.so"
+
+int umem_debug_level = 0;
+int umem_is_standalone = 0;
+
+/*ARGSUSED*/
+int
+umem_debug(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
+{
+	umem_debug_level ^= 1;
+
+	mdb_printf("umem: debugging is now %s\n",
+	    umem_debug_level ? "on" : "off");
+
+	return (DCMD_OK);
+}
+
+/*
+ * To further confuse the issue, this dmod can run against either
+ * libumem.so.1 *or* the libstandumem.so linked into kmdb(1M).  To figure
+ * out which one we are working against, we look up "umem_alloc" in both
+ * libumem.so and the executable.
+ *
+ * A further wrinkle is that libumem.so may not yet be loaded into the
+ * process' address space.  That can lead to either the lookup failing, or
+ * being unable to read from the data segment.  We treat either case as
+ * an error.
+ */
+int
+umem_set_standalone(void)
+{
+	GElf_Sym sym;
+	int ready;
+
+	if (mdb_lookup_by_obj(UMEM_OBJNAME, "umem_alloc", &sym) == 0)
+		umem_is_standalone = 0;
+	else if (mdb_lookup_by_obj(MDB_OBJ_EXEC, "umem_alloc", &sym) == 0)
+		umem_is_standalone = 1;
+	else
+		return (-1);
+
+	/*
+	 * now that we know where things should be, make sure we can actually
+	 * read things out.
+	 */
+	if (umem_readvar(&ready, "umem_ready") == -1)
+		return (-1);
+	return (0);
+}
+
+ssize_t
+umem_lookup_by_name(const char *name, GElf_Sym *sym)
+{
+	return (mdb_lookup_by_obj((umem_is_standalone ? MDB_OBJ_EXEC :
+	    UMEM_OBJNAME), name, sym));
+}
+
+/* This is like mdb_readvar, only for libumem.so's symbols */
+ssize_t
+umem_readvar(void *buf, const char *name)
+{
+	GElf_Sym sym;
+
+	if (umem_lookup_by_name(name, &sym))
+		return (-1);
+
+	if (mdb_vread(buf, sym.st_size, (uintptr_t)sym.st_value)
+	    == sym.st_size)
+		return ((ssize_t)sym.st_size);
+
+	return (-1);
+}
+
+int
+is_umem_sym(const char *sym, const char *prefix)
+{
+	char *tick_p = strrchr(sym, '`');
+
+	return (strncmp(sym, "libumem", 7) == 0 && tick_p != NULL &&
+	    strncmp(tick_p + 1, prefix, strlen(prefix)) == 0);
+}

umemdbg/mdb/common/misc.h

@@ -0,0 +1,67 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MDBMOD_MISC_H
+#define	_MDBMOD_MISC_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <mdb/mdb_modapi.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	offsetof(s, m)	((size_t)(&(((s *)0)->m)))
+
+extern int umem_debug(uintptr_t, uint_t, int, const mdb_arg_t *);
+
+extern int umem_set_standalone(void);
+extern ssize_t umem_lookup_by_name(const char *, GElf_Sym *);
+extern ssize_t umem_readvar(void *, const char *);
+
+/*
+ * Returns non-zero if sym matches libumem*`prefix*
+ */
+int is_umem_sym(const char *, const char *);
+
+#define	dprintf(x) if (umem_debug_level) { \
+	mdb_printf("umem debug: ");  \
+	/*CSTYLED*/\
+	mdb_printf x ;\
+}
+
+#define	dprintf_cont(x) if (umem_debug_level) { \
+	/*CSTYLED*/\
+	mdb_printf x ;\
+}
+
+extern int umem_debug_level;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _MDBMOD_MISC_H */

umemdbg/mdb/common/proc_kludges.c

@@ -0,0 +1,165 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2000-2002 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <mdb/mdb_modapi.h>
+#include <sys/machelf.h>
+
+#include <libproc.h>
+#include <stdio.h>
+
+#include "proc_kludges.h"
+
+typedef struct prockuldge_mappings {
+	struct ps_prochandle *pkm_Pr;
+
+	uint_t pkm_idx;
+
+	uint_t pkm_count;
+	uint_t pkm_max;
+
+	prmap_t *pkm_mappings;
+
+	uint_t pkm_old_max;
+	prmap_t *pkm_old_mappings;
+} prockludge_mappings_t;
+
+/* ARGSUSED */
+static int
+prockludge_mappings_iter(prockludge_mappings_t *pkm, const prmap_t *pmp,
+    const char *object_name)
+{
+	if (pkm->pkm_count >= pkm->pkm_max) {
+		int s = pkm->pkm_max ? pkm->pkm_max * 2 : 16;
+
+		pkm->pkm_old_max = pkm->pkm_max;
+		pkm->pkm_old_mappings = pkm->pkm_mappings;
+		pkm->pkm_max = s;
+		pkm->pkm_mappings = mdb_alloc(sizeof (prmap_t) * s, UM_SLEEP);
+
+		bcopy(pkm->pkm_old_mappings, pkm->pkm_mappings,
+		    sizeof (prmap_t) * pkm->pkm_old_max);
+
+		mdb_free(pkm->pkm_old_mappings,
+		    sizeof (prmap_t) * pkm->pkm_old_max);
+
+		pkm->pkm_old_mappings = NULL;
+		pkm->pkm_old_max = 0;
+	}
+	bcopy(pmp, &pkm->pkm_mappings[pkm->pkm_count++], sizeof (prmap_t));
+
+	return (0);
+}
+
+int
+prockludge_mappings_walk_init(mdb_walk_state_t *mws)
+{
+	struct ps_prochandle *Pr;
+	int rc;
+
+	prockludge_mappings_t *pkm;
+
+	if (mdb_get_xdata("pshandle", &Pr, sizeof (Pr)) == -1) {
+		mdb_warn("couldn't read pshandle xdata");
+		return (WALK_ERR);
+	}
+
+	pkm = mdb_zalloc(sizeof (prockludge_mappings_t), UM_SLEEP);
+	pkm->pkm_Pr = Pr;
+	mws->walk_data = pkm;
+
+	rc = Pmapping_iter(Pr, (proc_map_f *)prockludge_mappings_iter, pkm);
+	if (rc != 0) {
+		mdb_warn("Pmapping_iter failed");
+		/* clean up */
+		prockludge_mappings_walk_fini(mws);
+		return (WALK_ERR);
+	}
+	return (WALK_NEXT);
+}
+
+int
+prockludge_mappings_walk_step(mdb_walk_state_t *wsp)
+{
+	prockludge_mappings_t *pkm = wsp->walk_data;
+	int status;
+
+	if (pkm->pkm_idx >= pkm->pkm_count)
+		return (WALK_DONE);
+
+	status = wsp->walk_callback(0, &pkm->pkm_mappings[pkm->pkm_idx++],
+	    wsp->walk_cbdata);
+
+	return (status);
+}
+
+void
+prockludge_mappings_walk_fini(mdb_walk_state_t *wsp)
+{
+	prockludge_mappings_t *pkm = wsp->walk_data;
+	if (pkm != NULL) {
+		if (pkm->pkm_old_mappings != NULL) {
+			mdb_free(pkm->pkm_old_mappings,
+			    sizeof (prmap_t) * pkm->pkm_old_max);
+		}
+		if (pkm->pkm_mappings &&
+		    pkm->pkm_mappings != pkm->pkm_old_mappings) {
+			mdb_free(pkm->pkm_mappings,
+			    sizeof (prmap_t) * pkm->pkm_max);
+		}
+		mdb_free(pkm, sizeof (prockludge_mappings_t));
+	}
+}
+
+static int add_count = 0;
+
+void
+prockludge_add_walkers(void)
+{
+	mdb_walker_t w;
+
+	if (add_count++ == 0) {
+		w.walk_name = KLUDGE_MAPWALK_NAME;
+		w.walk_descr = "kludge: walk the process' prmap_ts";
+		w.walk_init = prockludge_mappings_walk_init;
+		w.walk_step = prockludge_mappings_walk_step;
+		w.walk_fini = prockludge_mappings_walk_fini;
+		w.walk_init_arg = NULL;
+
+		if (mdb_add_walker(&w) == -1) {
+			mdb_warn("unable to add walker "KLUDGE_MAPWALK_NAME);
+		}
+	}
+}
+
+void
+prockludge_remove_walkers(void)
+{
+	if (--add_count == 0) {
+		mdb_remove_walker(KLUDGE_MAPWALK_NAME);
+	}
+}

umemdbg/mdb/common/proc_kludges.h

@@ -0,0 +1,49 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2000-2002 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_PROC_KLUDGES_H
+#define	_PROC_KLUDGES_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#define	KLUDGE_MAPWALK_NAME "__prockludge_mappings"
+
+extern int prockludge_mappings_walk_init(mdb_walk_state_t *);
+extern int prockludge_mappings_walk_step(mdb_walk_state_t *);
+extern void prockludge_mappings_walk_fini(mdb_walk_state_t *);
+
+extern void prockludge_add_walkers(void);
+extern void prockludge_remove_walkers(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _PROC_KLUDGES_H */

umemdbg/mdb/common/umem.h

@@ -0,0 +1,140 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_MDBMOD_UMEM_H
+#define	_MDBMOD_UMEM_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <mdb/mdb_modapi.h>
+#include <umem_impl.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int umem_ready;
+extern uint32_t umem_stack_depth;
+
+extern int umem_cache_walk_init(mdb_walk_state_t *);
+extern int umem_cache_walk_step(mdb_walk_state_t *);
+extern void umem_cache_walk_fini(mdb_walk_state_t *);
+
+extern int umem_cpu_walk_init(mdb_walk_state_t *);
+extern int umem_cpu_walk_step(mdb_walk_state_t *);
+extern void umem_cpu_walk_fini(mdb_walk_state_t *);
+
+extern int umem_cpu_cache_walk_init(mdb_walk_state_t *);
+extern int umem_cpu_cache_walk_step(mdb_walk_state_t *);
+
+extern int umem_slab_walk_init(mdb_walk_state_t *);
+extern int umem_slab_walk_partial_init(mdb_walk_state_t *);
+extern int umem_slab_walk_step(mdb_walk_state_t *);
+
+extern int umem_hash_walk_init(mdb_walk_state_t *wsp);
+extern int umem_hash_walk_step(mdb_walk_state_t *wsp);
+extern void umem_hash_walk_fini(mdb_walk_state_t *wsp);
+
+extern int umem_walk_init(mdb_walk_state_t *);
+extern int bufctl_walk_init(mdb_walk_state_t *);
+extern int freemem_walk_init(mdb_walk_state_t *);
+extern int freectl_walk_init(mdb_walk_state_t *);
+
+extern int umem_walk_step(mdb_walk_state_t *);
+extern void umem_walk_fini(mdb_walk_state_t *);
+
+extern int bufctl_history_walk_init(mdb_walk_state_t *);
+extern int bufctl_history_walk_step(mdb_walk_state_t *);
+extern void bufctl_history_walk_fini(mdb_walk_state_t *);
+
+extern int allocdby_walk_init(mdb_walk_state_t *);
+extern int allocdby_walk_step(mdb_walk_state_t *);
+extern void allocdby_walk_fini(mdb_walk_state_t *);
+
+extern int freedby_walk_init(mdb_walk_state_t *);
+extern int freedby_walk_step(mdb_walk_state_t *);
+extern void freedby_walk_fini(mdb_walk_state_t *);
+
+extern int umem_log_walk_init(mdb_walk_state_t *);
+extern int umem_log_walk_step(mdb_walk_state_t *);
+extern void umem_log_walk_fini(mdb_walk_state_t *);
+
+extern int allocdby_walk_init(mdb_walk_state_t *);
+extern int allocdby_walk_step(mdb_walk_state_t *);
+extern void allocdby_walk_fini(mdb_walk_state_t *);
+
+extern int freedby_walk_init(mdb_walk_state_t *);
+extern int freedby_walk_step(mdb_walk_state_t *);
+extern void freedby_walk_fini(mdb_walk_state_t *);
+
+extern int vmem_walk_init(mdb_walk_state_t *);
+extern int vmem_walk_step(mdb_walk_state_t *);
+extern void vmem_walk_fini(mdb_walk_state_t *);
+
+extern int vmem_postfix_walk_step(mdb_walk_state_t *);
+
+extern int vmem_seg_walk_init(mdb_walk_state_t *);
+extern int vmem_seg_walk_step(mdb_walk_state_t *);
+extern void vmem_seg_walk_fini(mdb_walk_state_t *);
+
+extern int vmem_span_walk_init(mdb_walk_state_t *);
+extern int vmem_alloc_walk_init(mdb_walk_state_t *);
+extern int vmem_free_walk_init(mdb_walk_state_t *);
+
+extern int allocdby(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int bufctl(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int bufctl_audit(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int freedby(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umalog(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umausers(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_cache(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_log(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_malloc_dist(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_malloc_info(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_status(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_verify(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_verify_alloc(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int umem_verify_free(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int vmem(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int vmem_seg(uintptr_t, uint_t, int, const mdb_arg_t *);
+extern int whatis(uintptr_t, uint_t, int, const mdb_arg_t *);
+
+extern void bufctl_help(void);
+extern void umem_malloc_dist_help(void);
+extern void umem_malloc_info_help(void);
+extern void vmem_seg_help(void);
+
+/*
+ * utility functions for the rest of libumem
+ */
+extern int umem_init(void);
+extern int umem_get_magsize(const umem_cache_t *);
+extern size_t umem_estimate_allocated(uintptr_t, const umem_cache_t *);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _MDBMOD_UMEM_H */

umemdbg/mdb/common/umem_pagesize.h

@@ -0,0 +1,44 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2002 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_UMEM_PAGESIZE_H
+#define	_UMEM_PAGESIZE_H
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+extern size_t umem_pagesize;
+#undef PAGESIZE
+#define	PAGESIZE (umem_pagesize)
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _UMEM_PAGESIZE_H */

vmem.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem.c	1.10	05/06/08 SMI" */

vmem_base.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_base.c	1.6	05/06/08 SMI" */

vmem_base.h

@@ -2,8 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, (the "License").
-   You may not use this file except in compliance with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -22,7 +22,7 @@
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2012 Joyent, Inc. All rights reserved.
+ * Copyright 2012 Joyent, Inc. All rights reserved.
  */
 
 #ifndef	_VMEM_BASE_H

vmem_mmap.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_mmap.c	1.2	05/06/08 SMI" */

vmem_sbrk.c

@@ -23,7 +23,7 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  *
- * Portions Copyright 2006-2008 Message Systems, Inc. All rights reserved.
+ * Copyright 2006-2008 Message Systems, Inc. All rights reserved.
  */
 
 /* #pragma ident	"@(#)vmem_sbrk.c	1.4	05/06/08 SMI" */