Add a qsort variant optimized for integers and use in global dedupe.

Cleanup LZMA CRC64/32 declarations and add a header.
Fix heapq header.
This commit is contained in:
Moinak Ghosh 2013-05-03 22:06:55 +05:30
parent c43e99f422
commit 0cf94c308a
7 changed files with 364 additions and 18 deletions

View file

@ -85,7 +85,7 @@ BZLIB_OBJS = $(BZLIB_SRCS:.c=.o)
BZLIB_CPPFLAGS = @LIBBZ2_INC@ BZLIB_CPPFLAGS = @LIBBZ2_INC@
RABINSRCS = rabin/rabin_dedup.c rabin/global/index.c rabin/global/dedupe_config.c RABINSRCS = rabin/rabin_dedup.c rabin/global/index.c rabin/global/dedupe_config.c
RABINHDRS = rabin/rabin_dedup.h utils/utils.h rabin/global/index.h rabin/global/dedupe_config.h RABINHDRS = rabin/rabin_dedup.h utils/utils.h rabin/global/index.h rabin/global/dedupe_config.h lzma/lzma_crc.h utils/qsort.h
RABINOBJS = $(RABINSRCS:.c=.o) RABINOBJS = $(RABINSRCS:.c=.o)
BSDIFFSRCS = bsdiff/bsdiff.c bsdiff/bspatch.c bsdiff/rle_encoder.c BSDIFFSRCS = bsdiff/bsdiff.c bsdiff/bspatch.c bsdiff/rle_encoder.c
@ -112,7 +112,7 @@ PPMDOBJS = $(PPMDSRCS:.c=.o)
CRCSRCS = lzma/crc64_fast.c lzma/crc64_table.c lzma/crc32_fast.c lzma/crc32_table.c CRCSRCS = lzma/crc64_fast.c lzma/crc64_table.c lzma/crc32_fast.c lzma/crc32_table.c
CRCHDRS = lzma/crc64_table_le.h lzma/crc64_table_be.h lzma/crc_macros.h \ CRCHDRS = lzma/crc64_table_le.h lzma/crc64_table_be.h lzma/crc_macros.h \
lzma/crc32_table_le.h lzma/crc32_table_be.h lzma/crc32_table_le.h lzma/crc32_table_be.h lzma/lzma_crc.h
CRCOBJS = $(CRCSRCS:.c=.o) CRCOBJS = $(CRCSRCS:.c=.o)
LZPSRCS = lzp/lzp.c LZPSRCS = lzp/lzp.c

View file

@ -41,7 +41,7 @@
// //
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
#include <crc_macros.h> #include <lzma_crc.h>
extern const uint32_t lzma_crc32_table[8][256]; extern const uint32_t lzma_crc32_table[8][256];

View file

@ -39,7 +39,7 @@
// //
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
#include <crc_macros.h> #include "lzma_crc.h"
#ifdef WORDS_BIGENDIAN #ifdef WORDS_BIGENDIAN
# define A1(x) ((x) >> 56) # define A1(x) ((x) >> 56)

35
lzma/lzma_crc.h Normal file
View file

@ -0,0 +1,35 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
*/
#ifndef __LZMA_CRC_H__
#define __LZMA_CRC_H__
#include <crc_macros.h>
uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc);
uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc);
#endif

View file

@ -75,6 +75,8 @@
#include <pthread.h> #include <pthread.h>
#include <heapq.h> #include <heapq.h>
#include <xxhash.h> #include <xxhash.h>
#include <qsort.h>
#include <lzma_crc.h>
#include "rabin_dedup.h" #include "rabin_dedup.h"
#if defined(__USE_SSE_INTRIN__) && defined(__SSE4_1__) && RAB_POLYNOMIAL_WIN_SIZE == 16 #if defined(__USE_SSE_INTRIN__) && defined(__SSE4_1__) && RAB_POLYNOMIAL_WIN_SIZE == 16
@ -102,7 +104,6 @@ extern int bsdiff(u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t newsi
extern bsize_t get_bsdiff_sz(u_char *pbuf); extern bsize_t get_bsdiff_sz(u_char *pbuf);
extern int bspatch(u_char *pbuf, u_char *oldbuf, bsize_t oldsize, u_char *newbuf, extern int bspatch(u_char *pbuf, u_char *oldbuf, bsize_t oldsize, u_char *newbuf,
bsize_t *_newsize); bsize_t *_newsize);
extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc);
static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
uint64_t ir[256], out[256]; uint64_t ir[256], out[256];
@ -423,20 +424,14 @@ isort_uint64(uint64_t *ary, uint32_t nitems)
} }
/* /*
* Callback for qsort() for 64-bit min-values list in hash values. * Sort an array of 64-bit unsigned integers. The QSORT macro provides an
* inline quicksort routine that does not use a callback function.
*/ */
int #define int_lt(a,b) ((*a)<(*b))
cmpint(const void *a, const void *b) static void
do_qsort(uint64_t *arr, uint32_t len)
{ {
uint64_t a1 = *((uint64_t *)a); QSORT(uint64_t, arr, len, int_lt);
uint64_t b1 = *((uint64_t *)b);
if (a1 < b1)
return (-1);
else if (a1 == b1)
return (0);
else
return (1);
} }
static inline int static inline int
@ -836,6 +831,7 @@ process_blocks:
length = 0; length = 0;
dedupe_index_sz++; dedupe_index_sz++;
} }
/* /*
* Add a reference entry to the dedupe array. * Add a reference entry to the dedupe array.
*/ */
@ -919,7 +915,7 @@ process_blocks:
* Sort concatenated chunk hash buffer by raw 64-bit integer * Sort concatenated chunk hash buffer by raw 64-bit integer
* magnitudes. * magnitudes.
*/ */
qsort(seg_heap, length/8, 8, cmpint); do_qsort((uint64_t *)seg_heap, length/8);
/* /*
* Compute the min-values range similarity hashes. * Compute the min-values range similarity hashes.

View file

@ -24,6 +24,7 @@
#ifndef __HEAPQ_H_ #ifndef __HEAPQ_H_
#define __HEAPQ_H_
#define __TYPE int64_t #define __TYPE int64_t
typedef struct { typedef struct {

314
utils/qsort.h Normal file
View file

@ -0,0 +1,314 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*/
/* $Id: qsort.h,v 1.5 2008-01-28 18:16:49 mjt Exp $
* Adopted from GNU glibc by Mjt.
* See stdlib/qsort.c in glibc */
/* Copyright (C) 1991, 1992, 1996, 1997, 1999 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Written by Douglas C. Schmidt (schmidt@ics.uci.edu).
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, write to the Free
Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
02111-1307 USA. */
/* in-line qsort implementation. Differs from traditional qsort() routine
* in that it is a macro, not a function, and instead of passing an address
* of a comparison routine to the function, it is possible to inline
* comparison routine, thus speeding up sorting a lot.
*
* Usage:
* #include "iqsort.h"
* #define islt(a,b) (strcmp((*a),(*b))<0)
* char *arr[];
* int n;
* QSORT(char*, arr, n, islt);
*
* The "prototype" and 4 arguments are:
* QSORT(TYPE,BASE,NELT,ISLT)
* 1) type of each element, TYPE,
* 2) address of the beginning of the array, of type TYPE*,
* 3) number of elements in the array, and
* 4) comparision routine.
* Array pointer and number of elements are referenced only once.
* This is similar to a call
* qsort(BASE,NELT,sizeof(TYPE),ISLT)
* with the difference in last parameter.
* Note the islt macro/routine (it receives pointers to two elements):
* the only condition of interest is whenever one element is less than
* another, no other conditions (greather than, equal to etc) are tested.
* So, for example, to define integer sort, use:
* #define islt(a,b) ((*a)<(*b))
* QSORT(int, arr, n, islt)
*
* The macro could be used to implement a sorting function (see examples
* below), or to implement the sorting algorithm inline. That is, either
* create a sorting function and use it whenever you want to sort something,
* or use QSORT() macro directly instead a call to such routine. Note that
* the macro expands to quite some code (compiled size of int qsort on x86
* is about 700..800 bytes).
*
* Using this macro directly it isn't possible to implement traditional
* qsort() routine, because the macro assumes sizeof(element) == sizeof(TYPE),
* while qsort() allows element size to be different.
*
* Several ready-to-use examples:
*
* Sorting array of integers:
* void int_qsort(int *arr, unsigned n) {
* #define int_lt(a,b) ((*a)<(*b))
* QSORT(int, arr, n, int_lt);
* }
*
* Sorting array of string pointers:
* void str_qsort(char *arr[], unsigned n) {
* #define str_lt(a,b) (strcmp((*a),(*b)) < 0)
* QSORT(char*, arr, n, str_lt);
* }
*
* Sorting array of structures:
*
* struct elt {
* int key;
* ...
* };
* void elt_qsort(struct elt *arr, unsigned n) {
* #define elt_lt(a,b) ((a)->key < (b)->key)
* QSORT(struct elt, arr, n, elt_lt);
* }
*
* And so on.
*/
/* Swap two items pointed to by A and B using temporary buffer t. */
#define _QSORT_SWAP(a, b, t) ((void)((t = *a), (*a = *b), (*b = t)))
/* Discontinue quicksort algorithm when partition gets below this size.
This particular magic number was chosen to work best on a Sun 4/260. */
#define _QSORT_MAX_THRESH 4
/* Stack node declarations used to store unfulfilled partition obligations
* (inlined in QSORT).
typedef struct {
QSORT_TYPE *_lo, *_hi;
} qsort_stack_node;
*/
/* The next 4 #defines implement a very fast in-line stack abstraction. */
/* The stack needs log (total_elements) entries (we could even subtract
log(MAX_THRESH)). Since total_elements has type unsigned, we get as
upper bound for log (total_elements):
bits per byte (CHAR_BIT) * sizeof(unsigned). */
#define _QSORT_STACK_SIZE (8 * sizeof(unsigned))
#define _QSORT_PUSH(top, low, high) \
(((top->_lo = (low)), (top->_hi = (high)), ++top))
#define _QSORT_POP(low, high, top) \
((--top, (low = top->_lo), (high = top->_hi)))
#define _QSORT_STACK_NOT_EMPTY (_stack < _top)
/* Order size using quicksort. This implementation incorporates
four optimizations discussed in Sedgewick:
1. Non-recursive, using an explicit stack of pointer that store the
next array partition to sort. To save time, this maximum amount
of space required to store an array of SIZE_MAX is allocated on the
stack. Assuming a 32-bit (64 bit) integer for size_t, this needs
only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes).
Pretty cheap, actually.
2. Chose the pivot element using a median-of-three decision tree.
This reduces the probability of selecting a bad pivot value and
eliminates certain extraneous comparisons.
3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving
insertion sort to order the MAX_THRESH items within each partition.
This is a big win, since insertion sort is faster for small, mostly
sorted array segments.
4. The larger of the two sub-partitions is always pushed onto the
stack first, with the algorithm then concentrating on the
smaller partition. This *guarantees* no more than log (total_elems)
stack size is needed (actually O(1) in this case)! */
/* The main code starts here... */
#define QSORT(QSORT_TYPE,QSORT_BASE,QSORT_NELT,QSORT_LT) \
{ \
QSORT_TYPE *const _base = (QSORT_BASE); \
const unsigned _elems = (QSORT_NELT); \
QSORT_TYPE _hold; \
\
/* Don't declare two variables of type QSORT_TYPE in a single \
* statement: eg `TYPE a, b;', in case if TYPE is a pointer, \
* expands to `type* a, b;' wich isn't what we want. \
*/ \
\
if (_elems > _QSORT_MAX_THRESH) { \
QSORT_TYPE *_lo = _base; \
QSORT_TYPE *_hi = _lo + _elems - 1; \
struct { \
QSORT_TYPE *_hi; QSORT_TYPE *_lo; \
} _stack[_QSORT_STACK_SIZE], *_top = _stack + 1; \
\
while (_QSORT_STACK_NOT_EMPTY) { \
QSORT_TYPE *_left_ptr; QSORT_TYPE *_right_ptr; \
\
/* Select median value from among LO, MID, and HI. Rearrange \
LO and HI so the three values are sorted. This lowers the \
probability of picking a pathological pivot value and \
skips a comparison for both the LEFT_PTR and RIGHT_PTR in \
the while loops. */ \
\
QSORT_TYPE *_mid = _lo + ((_hi - _lo) >> 1); \
\
if (QSORT_LT (_mid, _lo)) \
_QSORT_SWAP (_mid, _lo, _hold); \
if (QSORT_LT (_hi, _mid)) { \
_QSORT_SWAP (_mid, _hi, _hold); \
if (QSORT_LT (_mid, _lo)) \
_QSORT_SWAP (_mid, _lo, _hold); \
} \
\
_left_ptr = _lo + 1; \
_right_ptr = _hi - 1; \
\
/* Here's the famous ``collapse the walls'' section of quicksort. \
Gotta like those tight inner loops! They are the main reason \
that this algorithm runs much faster than others. */ \
do { \
while (QSORT_LT (_left_ptr, _mid)) \
++_left_ptr; \
\
while (QSORT_LT (_mid, _right_ptr)) \
--_right_ptr; \
\
if (_left_ptr < _right_ptr) { \
_QSORT_SWAP (_left_ptr, _right_ptr, _hold); \
if (_mid == _left_ptr) \
_mid = _right_ptr; \
else if (_mid == _right_ptr) \
_mid = _left_ptr; \
++_left_ptr; \
--_right_ptr; \
} \
else if (_left_ptr == _right_ptr) { \
++_left_ptr; \
--_right_ptr; \
break; \
} \
} while (_left_ptr <= _right_ptr); \
\
/* Set up pointers for next iteration. First determine whether \
left and right partitions are below the threshold size. If so, \
ignore one or both. Otherwise, push the larger partition's \
bounds on the stack and continue sorting the smaller one. */ \
\
if (_right_ptr - _lo <= _QSORT_MAX_THRESH) { \
if (_hi - _left_ptr <= _QSORT_MAX_THRESH) \
/* Ignore both small partitions. */ \
_QSORT_POP (_lo, _hi, _top); \
else \
/* Ignore small left partition. */ \
_lo = _left_ptr; \
} \
else if (_hi - _left_ptr <= _QSORT_MAX_THRESH) \
/* Ignore small right partition. */ \
_hi = _right_ptr; \
else if (_right_ptr - _lo > _hi - _left_ptr) { \
/* Push larger left partition indices. */ \
_QSORT_PUSH (_top, _lo, _right_ptr); \
_lo = _left_ptr; \
} \
else { \
/* Push larger right partition indices. */ \
_QSORT_PUSH (_top, _left_ptr, _hi); \
_hi = _right_ptr; \
} \
} \
} \
\
/* Once the BASE array is partially sorted by quicksort the rest \
is completely sorted using insertion sort, since this is efficient \
for partitions below MAX_THRESH size. BASE points to the \
beginning of the array to sort, and END_PTR points at the very \
last element in the array (*not* one beyond it!). */ \
\
{ \
QSORT_TYPE *const _end_ptr = _base + _elems - 1; \
QSORT_TYPE *_tmp_ptr = _base; \
register QSORT_TYPE *_run_ptr; \
QSORT_TYPE *_thresh; \
\
_thresh = _base + _QSORT_MAX_THRESH; \
if (_thresh > _end_ptr) \
_thresh = _end_ptr; \
\
/* Find smallest element in first threshold and place it at the \
array's beginning. This is the smallest array element, \
and the operation speeds up insertion sort's inner loop. */ \
\
for (_run_ptr = _tmp_ptr + 1; _run_ptr <= _thresh; ++_run_ptr) \
if (QSORT_LT (_run_ptr, _tmp_ptr)) \
_tmp_ptr = _run_ptr; \
\
if (_tmp_ptr != _base) \
_QSORT_SWAP (_tmp_ptr, _base, _hold); \
\
/* Insertion sort, running from left-hand-side \
* up to right-hand-side. */ \
\
_run_ptr = _base + 1; \
while (++_run_ptr <= _end_ptr) { \
_tmp_ptr = _run_ptr - 1; \
while (QSORT_LT (_run_ptr, _tmp_ptr)) \
--_tmp_ptr; \
\
++_tmp_ptr; \
if (_tmp_ptr != _run_ptr) { \
QSORT_TYPE *_trav = _run_ptr + 1; \
while (--_trav >= _run_ptr) { \
QSORT_TYPE *_hi; QSORT_TYPE *_lo; \
_hold = *_trav; \
\
for (_hi = _lo = _trav; --_lo >= _tmp_ptr; _hi = _lo) \
*_hi = *_lo; \
*_hi = _hold; \
} \
} \
} \
} \
\
}