greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add basic framework for file type based filters during libarchive stage.

Browse source 
Add packJPG filter for Jpeg files (not active yet).
Directory format changes for clarity.
This commit is contained in:
Moinak Ghosh 2013-11-10 23:09:42 +05:30
parent a5f1624a33
commit 75dfa6a6fb
22 changed files with 13255 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
Makefile.in
View file

@ -28,11 +28,9 @@ LINKLIB=pcompress
LIBVER=1
MAINSRCS = utils/utils.c allocator.c lzma_compress.c ppmd_compress.c \
adaptive_compress.c lzfx_compress.c lz4_compress.c none_compress.c \
utils/xxhash_base.c utils/heap.c utils/cpuid.c archive/pc_archive.c \
utils/phash/phash.c utils/phash/lookupa.c utils/phash/recycle.c pcompress.c
utils/xxhash_base.c utils/heap.c utils/cpuid.c pcompress.c
MAINHDRS = allocator.h pcompress.h utils/utils.h utils/xxhash.h utils/heap.h \
utils/cpuid.h utils/xxhash.h archive/pc_archive.h utils/phash/standard.h \
utils/phash/lookupa.h utils/phash/recycle.h utils/phash/phash.h
utils/cpuid.h utils/xxhash.h archive/pc_archive.h
MAINOBJS = $(MAINSRCS:.c=.o)
PROGSRCS = main.c
@ -124,14 +122,26 @@ CRCHDRS = lzma/crc64_table_le.h lzma/crc64_table_be.h lzma/crc_macros.h \
lzma/crc32_table_le.h lzma/crc32_table_be.h lzma/lzma_crc.h
CRCOBJS = $(CRCSRCS:.c=.o)
LZPSRCS = lzp/lzp.c
LZPHDRS = lzp/lzp.h
LZPSRCS = filters/lzp/lzp.c
LZPHDRS = filters/lzp/lzp.h
LZPOBJS = $(LZPSRCS:.c=.o)
DELTA2SRCS = delta2/delta2.c
DELTA2HDRS = delta2/delta2.h
DELTA2SRCS = filters/delta2/delta2.c
DELTA2HDRS = filters/delta2/delta2.h
DELTA2OBJS = $(DELTA2SRCS:.c=.o)
ARCHIVESRCS = archive/pc_archive.c archive/pc_arc_filter.c utils/phash/phash.c \
utils/phash/lookupa.c utils/phash/recycle.c
ARCHIVEHDRS = pcompress.h utils/utils.h archive/pc_archive.h utils/phash/standard.h \
utils/phash/lookupa.h utils/phash/recycle.h utils/phash/phash.h archive/pc_arc_filter.h
ARCHIVEOBJS = $(ARCHIVESRCS:.c=.o)
PJPGSRCS = filters/packjpg/aricoder.cpp filters/packjpg/bitops.cpp filters/packjpg/packjpg.cpp \
archive/pjpg_helper.cpp
PJPGHDRS = filters/packjpg/aricoder.h filters/packjpg/bitops.h filters/packjpg/dct8x8.h \
filters/packjpg/packjpglib.h filters/packjpg/pjpgtbl.h
PJPGOBJS = $(PJPGSRCS:.cpp=.o)
SKEIN_BLOCK_C = crypto/skein/skein_block.c
SKEIN_BLOCK_ASM = crypto/skein/skein_block_x64.s
SKEIN_BLOCK_SRC = @SKEIN_BLOCK@
@ -157,8 +167,8 @@ LIBBSCLIB = @LIBBSCLIB@
LIBBSCGEN_OPT = -fopenmp
LIBBSCCPPFLAGS = -I$(LIBBSCDIR)/libbsc -DENABLE_PC_LIBBSC
TRANSP_SRCS = transpose/transpose.c
TRANSP_HDRS = transpose/transpose.h
TRANSP_SRCS = filters/transpose/transpose.c
TRANSP_HDRS = filters/transpose/transpose.h
TRANSP_OBJS = $(TRANSP_SRCS:.c=.o)
KECCAK_SRC_COMMON = crypto/keccak/genKAT.c crypto/keccak/KeccakDuplex.c \
@ -182,21 +192,23 @@ KECCAK_HDRS = @KECCAK_HDRS@
KECCAK_OBJS = $(KECCAK_SRCS:.c=.o)
KECCAK_OBJS_ASM = $(KECCAK_SRCS_ASM:.s=.o)
BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~ crypto/sha2/*~ \
BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ filters/lzp/*~ utils/*~ crypto/sha2/*~ \
crypto/sha2/intel/*~ crypto/aes/*~ crypto/scrypt/*~ crypto/*~ rabin/global/*~ \
delta2/*~ crypto/keccak/*~ transpose/*~ crypto/skein/*~ crypto/keccak/*.o \
archive/*~
archive/*~ filters/delta2/*~ filters/packjpg/*~ filters/transpose/*~
RM = rm -f
RM_RF = rm -rf
COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \
BASE_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \
-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \
-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \
-I./filters/lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \
-I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ -I./rabin/global \
-I./crypto/keccak -I./transpose -I./crypto/blake2 $(EXTRA_CPPFLAGS) \
-I./crypto/xsalsa20 -I./archive -pedantic -Wall -std=gnu99 \
-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare \
@COMPAT_CPPFLAGS@ @XSALSA20_DEBUG@ -I@LIBARCHIVE_INC@
-I./crypto/keccak -I./filters/transpose -I./crypto/blake2 $(EXTRA_CPPFLAGS) \
-I./crypto/xsalsa20 -I./archive -pedantic -Wall -I./filters -fno-strict-aliasing \
-Wno-unused-but-set-variable -Wno-enum-compare \
@COMPAT_CPPFLAGS@ @XSALSA20_DEBUG@ -I@LIBARCHIVE_INC@ -I./filters/packjpg
COMMON_CPPFLAGS = $(BASE_CPPFLAGS) -std=gnu99
COMMON_CPPFLAGS_cpp = $(BASE_CPPFLAGS)
COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \
@ -206,7 +218,7 @@ OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \
$(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) $(DELTA2OBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) \
$(SKEIN_BLOCK_OBJ) @SHA2ASM_OBJS@ @SHA2_OBJS@ $(KECCAK_OBJS) $(KECCAK_OBJS_ASM) \
$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS) \
@CRYPTO_COMPAT_OBJS@ $(CRYPTO_ASM_OBJS)
@CRYPTO_COMPAT_OBJS@ $(CRYPTO_ASM_OBJS) $(ARCHIVEOBJS) $(PJPGOBJS)
DEBUG_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp -fPIC
DEBUG_COMPILE = gcc -g -c @EXTRA_OPT_FLAGS@ -fPIC
@ -277,6 +289,13 @@ $(LZPOBJS): $(LZPSRCS) $(LZPHDRS)
$(DELTA2OBJS): $(DELTA2SRCS) $(DELTA2HDRS)
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(ARCHIVEOBJS): $(ARCHIVESRCS) $(ARCHIVEHDRS)
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(PJPGOBJS): $(PJPGSRCS) $(PJPGHDRS)
$(COMPILE_cpp) $(COMMON_VEC_FLAGS) @SSE_OPT_FLAGS@ -O2 -fsched-spec-load \
$(VEC_FLAGS) -DBUILD_LIB $(COMMON_CPPFLAGS_cpp) $(@:.o=.cpp) -o $@
$(SKEIN_BLOCK_OBJ): $(SKEIN_BLOCK_SRC)
$(COMPILE) $(SKEIN_FLAGS) $(SKEIN_BLOCK_SRC) -o $@

130
archive/pc_arc_filter.c Normal file
View file

@ -0,0 +1,130 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
*/
/*
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <limits.h>
#include <utils.h>
#include <sys/mman.h>
#include <ctype.h>
#include <archive.h>
#include <archive_entry.h>
#include "pc_arc_filter.h"
#include "pc_archive.h"
#define PACKJPG_DEF_BUFSIZ (512 * 1024)
#define JPG_SIZE_LIMIT (100 * 1024 * 1024)
struct packjpg_filter_data {
uchar_t *buff;
size_t bufflen;
};
extern size_t packjpg_filter_process(uchar_t *in_buf, size_t len, uchar_t **out_buf);
int packjpg_filter(struct filter_info *fi, void *filter_private);
void
add_filters_by_ext()
{
struct packjpg_filter_data *pjdat;
pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data));
pjdat->buff = (uchar_t *)malloc(PACKJPG_DEF_BUFSIZ);
pjdat->bufflen = PACKJPG_DEF_BUFSIZ;
if (insert_filter_data(packjpg_filter, pjdat, "pjg") != 0) {
free(pjdat->buff);
free(pjdat);
log_msg(LOG_WARN, 0, "Failed to add filter module for packJPG.");
}
}
/* a short reminder about input/output stream types
for the pjglib_init_streams() function
if input is file
----------------
in_scr -> name of input file
in_type -> 0
in_size -> ignore
if input is memory
------------------
in_scr -> array containg data
in_type -> 1
in_size -> size of data array
if input is *FILE (f.e. stdin)
------------------------------
in_src -> stream pointer
in_type -> 2
in_size -> ignore
vice versa for output streams! */
int
packjpg_filter(struct filter_info *fi, void *filter_private)
{
struct packjpg_filter_data *pjdat = (struct packjpg_filter_data *)filter_private;
uchar_t *mapbuf, *out;
size_t len;
len = archive_entry_size(fi->entry);
if (len > JPG_SIZE_LIMIT) // Bork on massive JPEGs
return (-1);
mapbuf = mmap(NULL, len, PROT_READ, MAP_SHARED, fi->fd, 0);
if (mapbuf == NULL)
return (-1);
if (pjdat->bufflen < len) {
free(pjdat->buff);
pjdat->bufflen = len;
pjdat->buff = malloc(pjdat->bufflen);
if (pjdat->buff == NULL) {
log_msg(LOG_ERR, 1, "Out of memory.");
munmap(mapbuf, len);
return (-1);
}
}
/*
* Helper routine to bridge to packJPG C++ lib, without changing packJPG itself.
*/
out = pjdat->buff;
if ((len = packjpg_filter_process(mapbuf, len, &out)) == 0) {
return (-1);
}
return (archive_write_data(fi->target_arc, out, len));
}

53
archive/pc_arc_filter.h Normal file
View file

@ -0,0 +1,53 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
*/
#ifndef _PC_ARCHIVE_FILTER_H
#define _PC_ARCHIVE_FILTER_H
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <archive.h>
#include <archive_entry.h>
#ifdef __cplusplus
extern "C" {
#endif
struct filter_info {
struct archive *target_arc;
struct archive_entry *entry;
int fd;
};
typedef int (*filter_func_ptr)(struct filter_info *fi, void *filter_private);
void add_filters_by_ext();
#ifdef __cplusplus
}
#endif
#endif

137
archive/pc_archive.c
View file

@ -27,8 +27,8 @@
* This file includes all the archiving related functions. Pathnames are sorted
* based on extension (or first 4 chars of name if no extension) and size. A simple
* external merge sort is used. This sorting yields better compression ratio.
*
* Sorting is enabled for compression levels greater than 2.
*
* Sorting is enabled for compression levels greater than 6.
*/
#include <sys/types.h>
#include <sys/stat.h>
@ -45,10 +45,10 @@
#include <ctype.h>
#include <archive.h>
#include <archive_entry.h>
#include "pc_archive.h"
#include <phash/phash.h>
#include <phash/extensions.h>
#include <phash/standard.h>
#include "pc_archive.h"
#undef _FEATURES_H
#define _XOPEN_SOURCE 700
@ -60,6 +60,8 @@ pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
static struct ext_hash_entry {
uint64_t extnum;
int type;
void *filter_private;
filter_func_ptr filter_func;
} *exthtab = NULL;
/*
@ -809,12 +811,11 @@ setup_extractor(pc_ctx_t *pctx)
}
/*
* Routines to archive members and write the archive to pipe. Most of the following
* code is adapted from some of the Libarchive bsdtar code.
* Routines to archive members and write the file data to the callback. Portions of
* the following code is adapted from some of the Libarchive bsdtar code.
*/
static int
copy_file_data(pc_ctx_t *pctx, struct archive *arc,
struct archive *in_arc, struct archive_entry *entry, int typ)
copy_file_data(pc_ctx_t *pctx, struct archive *arc, struct archive_entry *entry, int typ)
{
size_t sz, offset, len;
ssize_t bytes_to_write;
@ -833,6 +834,10 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc,
return (-1);
}
/*
* Use mmap for copying file data. Not necessarily for performance, but it saves on
* resident memory use.
*/
while (bytes_to_write > 0) {
uchar_t *src;
size_t wlen;
@ -843,6 +848,12 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc,
else
len = MMAP_SIZE;
mapbuf = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, offset);
if (mapbuf == NULL) {
/* Mmap failed; this is bad. */
log_msg(LOG_ERR, 1, "Mmap failed for %s.", fpath);
rv = -1;
break;
}
offset += len;
src = mapbuf;
wlen = len;
@ -852,7 +863,7 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc,
/*
* Write the entire mmap-ed buffer. Since we are writing to the compressor
* stage pipe there is no need for blocking.
* stage there is no need for blocking.
*/
wrtn = archive_write_data(arc, src, wlen);
if (wrtn < wlen) {
@ -870,8 +881,7 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc,
}
static int
write_entry(pc_ctx_t *pctx, struct archive *arc, struct archive *in_arc,
struct archive_entry *entry, int typ)
write_entry(pc_ctx_t *pctx, struct archive *arc, struct archive_entry *entry, int typ)
{
int rv;
@ -888,7 +898,7 @@ write_entry(pc_ctx_t *pctx, struct archive *arc, struct archive *in_arc,
}
if (archive_entry_size(entry) > 0) {
return (copy_file_data(pctx, arc, in_arc, entry, typ));
return (copy_file_data(pctx, arc, entry, typ));
}
return (0);
@ -999,7 +1009,7 @@ archiver_thread_func(void *dat) {
archive_entry_linkify(resolver, &entry, &spare_entry);
ent = entry;
while (ent != NULL) {
if (write_entry(pctx, arc, ard, ent, typ) != 0) {
if (write_entry(pctx, arc, ent, typ) != 0) {
goto done;
}
ent = spare_entry;
@ -1037,58 +1047,58 @@ start_archiver(pc_ctx_t *pctx) {
static int
copy_data_out(struct archive *ar, struct archive *aw)
{
int64_t offset;
const void *buff;
size_t size;
int r;
int64_t offset;
const void *buff;
size_t size;
int r;
for (;;) {
r = archive_read_data_block(ar, &buff, &size, &offset);
if (r == ARCHIVE_EOF)
return (ARCHIVE_OK);
if (r != ARCHIVE_OK)
return (r);
r = (int)archive_write_data_block(aw, buff, size, offset);
if (r < ARCHIVE_WARN)
r = ARCHIVE_WARN;
if (r != ARCHIVE_OK) {
archive_set_error(ar, archive_errno(aw),
"%s", archive_error_string(aw));
return (r);
}
}
for (;;) {
r = archive_read_data_block(ar, &buff, &size, &offset);
if (r == ARCHIVE_EOF)
return (ARCHIVE_OK);
if (r != ARCHIVE_OK)
return (r);
r = (int)archive_write_data_block(aw, buff, size, offset);
if (r < ARCHIVE_WARN)
r = ARCHIVE_WARN;
if (r != ARCHIVE_OK) {
archive_set_error(ar, archive_errno(aw),
"%s", archive_error_string(aw));
return (r);
}
}
}
static int
archive_extract_entry(struct archive *a, struct archive_entry *entry,
struct archive *ad)
{
int r, r2;
r = archive_write_header(ad, entry);
if (r < ARCHIVE_WARN)
r = ARCHIVE_WARN;
if (r != ARCHIVE_OK)
/* If _write_header failed, copy the error. */
archive_copy_error(a, ad);
else if (!archive_entry_size_is_set(entry) || archive_entry_size(entry) > 0)
/* Otherwise, pour data into the entry. */
r = copy_data_out(a, ad);
r2 = archive_write_finish_entry(ad);
if (r2 < ARCHIVE_WARN)
r2 = ARCHIVE_WARN;
/* Use the first message. */
if (r2 != ARCHIVE_OK && r == ARCHIVE_OK)
archive_copy_error(a, ad);
/* Use the worst error return. */
if (r2 < r)
r = r2;
return (r);
int r, r2;
r = archive_write_header(ad, entry);
if (r < ARCHIVE_WARN)
r = ARCHIVE_WARN;
if (r != ARCHIVE_OK)
/* If _write_header failed, copy the error. */
archive_copy_error(a, ad);
else if (!archive_entry_size_is_set(entry) || archive_entry_size(entry) > 0)
/* Otherwise, pour data into the entry. */
r = copy_data_out(a, ad);
r2 = archive_write_finish_entry(ad);
if (r2 < ARCHIVE_WARN)
r2 = ARCHIVE_WARN;
/* Use the first message. */
if (r2 != ARCHIVE_OK && r == ARCHIVE_OK)
archive_copy_error(a, ad);
/* Use the worst error return. */
if (r2 < r)
r = r2;
return (r);
}
/*
* Extract Thread function. Read an uncompressed archive from the pipe and extract
* members to disk. The decompressor writes to the other end of the pipe.
* Extract Thread function. Read an uncompressed archive from the decompressor stage
* and extract members to disk.
*/
static void *
extractor_thread_func(void *dat) {
@ -1241,7 +1251,11 @@ init_archive_mod() {
extnum = (extnum << 1) | extlist[i].ext[j];
exthtab[slot].extnum = extnum;
exthtab[slot].type = extlist[i].type;
exthtab[slot].filter_func = NULL;
exthtab[slot].filter_private = NULL;
}
add_filters_by_ext();
inited = 1;
} else {
rv = 1;
@ -1270,7 +1284,7 @@ detect_type_by_ext(char *path, int pathlen)
if (len == 0) goto out; // If extension is empty give up
ext = &path[i+1];
slot = phash(ext, len);
if (slot > PHASHNKEYS) goto out; // Extension maps outside hash table range, give up
if (slot >= PHASHNKEYS) goto out; // Extension maps outside hash table range, give up
extnum = 0;
/*
@ -1326,3 +1340,16 @@ detect_type_by_data(uchar_t *buf, size_t len)
return (TYPE_UNKNOWN);
}
int
insert_filter_data(filter_func_ptr func, void *filter_private, const char *ext)
{
ub4 slot = phash(ext, strlen(ext));
if (slot >= PHASHNKEYS || slot < 0) {
log_msg(LOG_WARN, 0, "Cannot add filter for unknown extension: %s", ext);
return (-1);
}
exthtab[slot].filter_func = func;
exthtab[slot].filter_private = filter_private;
return (0);
}

6
archive/pc_archive.h
View file

@ -23,13 +23,14 @@
*
*/
#ifndef _ARCHIVE_H
#define _ARCHIVE_H
#ifndef _PC_ARCHIVE_H
#define _PC_ARCHIVE_H
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <pcompress.h>
#include <pc_arc_filter.h>
#ifdef __cplusplus
extern "C" {
@ -52,6 +53,7 @@ int64_t archiver_read(void *ctx, void *buf, uint64_t count);
int64_t archiver_write(void *ctx, void *buf, uint64_t count);
int archiver_close(void *ctx);
int init_archive_mod();
int insert_filter_data(filter_func_ptr func, void *filter_private, const char *ext);
#ifdef __cplusplus
}

58
archive/pjpg_helper.cpp Normal file
View file

@ -0,0 +1,58 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
*/
/*
*/
#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <packjpglib.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef unsigned char uchar_t;
size_t
packjpg_filter_process(uchar_t *in_buf, size_t len, uchar_t **out_buf)
{
unsigned int len1;
pjglib_init_streams(in_buf, 1, len, *out_buf, 1);
len1 = len;
if (!pjglib_convert_stream2mem(out_buf, &len1, NULL))
return (0);
if (len1 == len)
return (0);
return (len1);
}
#ifdef __cplusplus
}
#endif

0
delta2/delta2.c → filters/delta2/delta2.c
View file

0
delta2/delta2.h → filters/delta2/delta2.h
View file

0
lzp/lzp.c → filters/lzp/lzp.c
View file

0
lzp/lzp.h → filters/lzp/lzp.h
View file

256
filters/packjpg/Readme.txt Normal file
View file

@ -0,0 +1,256 @@
packJPG v2.5g (09/14/2013)
~~~~~~~~~~~~~~~~~~~~~~~~~~
packJPG is a compression program specially designed for further
compression of JPEG images without causing any further loss. Typically
it reduces the file size of a JPEG file by 20%.
LGPL v3 license and special permissions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
All programs in this package are free software; you can redistribute
them and/or modify them under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either version 3
of the License, or (at your option) any later version.
The package is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
General Public License for more details at
http://www.gnu.org/copyleft/lgpl.html.
If the LGPL v3 license is not compatible with your software project you
might contact us and ask for a special permission to use the packJPG
library under different conditions. In any case, usage of the packJPG
algorithm under the LGPL v3 or above is highly advised and special
permissions will only be given where necessary on a case by case basis.
This offer is aimed mainly at closed source freeware developers seeking
to add PJG support to their software projects.
Copyright 2006...2013 by HTW Aalen University and Matthias Stirner.
Usage of packJPG
~~~~~~~~~~~~~~~~
JPEG files are compressed and PJG files are decompressed using this
command:
"packJPG [file(s)]"
packJPG recognizes file types on its own and decides whether to compress
(JPG) or decompress (PJG). For unrecognized file types no action is
taken. Files are recognized by content, not by extension.
packJPG supports wildcards like "*.*" and drag and drop of multiple
files. Filenames for output files are created automatically. In default
mode, files are never overwritten. If a filename is already in use,
packJPG creates a new filename by adding underscores.
If "-" is used as a filename input from stdin is assumed and output is
written to stdout. This can be useful for example if jpegtran is to be
used as a preprocessor.
Usage examples:
"packJPG *.pjg"
"packJPG lena.jpg"
"packJPG kodim??.jpg"
"packJPG - < sail.pjg > sail.jpg"
Command line switches
~~~~~~~~~~~~~~~~~~~~~
-ver verify files after processing
-v? level of verbosity; 0,1 or 2 is allowed (default 0)
-np no pause after processing files
-o overwrite existing files
-p proceed on warnings
-d discard meta-info
By default, compression is cancelled on warnings. If warnings are
skipped by using "-p", most files with warnings can also be compressed,
but JPEG files reconstructed from PJG files might not be bitwise
identical with the original JPEG files. There won't be any loss to
image data or quality however.
Unnecessary meta information can be discarded using "-d". This reduces
compressed files' sizes. Be warned though, reconstructed files won't be
bitwise identical with the original files and meta information will be
lost forever. As with "-p" there won't be any loss to image data or
quality.
There is no known case in which a file compressed by packJPG (without
the "-p" option, see above) couldn't be reconstructed to exactly the
state it was before. If you want an additional layer of safety you can
also use the verify option "-ver". In this mode, files are compressed,
then decompressed and the decompressed file compared to the original
file. If this test doesn't pass there will be an error message and the
compressed file won't be written to the drive.
Please note that the "-ver" option should never be used in conjunction
with the "-d" and/or "-p" options. As stated above, the "-p" and "-d"
options will most likely lead to reconstructed JPG files not being
bitwise identical to the original JPG files. In turn, the verification
process may fail on various files although nothing actually went wrong.
Usage examples:
"packJPG -v1 -o baboon.pjg"
"packJPG -ver lena.jpg"
"packJPG -d tiffany.jpg"
"packJPG -p *.jpg"
Known Limitations
~~~~~~~~~~~~~~~~~
packJPG is a compression program specially for JPEG files, so it doesn't
compress other file types.
packJPG has low error tolerance. JPEG files might not work with packJPG
even if they work perfectly with other image processing software. The
command line switch "-p" can be used to increase error tolerance and
compatibility.
If you try to drag and drop to many files at once, there might be a
windowed error message about missing privileges. In that case you can
try it again with less files or consider using the command prompt.
packJPG has been tested to work perfectly with thousands of files from
the command line. This issue also happens with drag and drop in other
applications, so it might not be a limitation of packJPG but a
limitation of Windows.
Compressed PJG files are not compatible between different packJPG
versions. You will get an error message if you try to decompress PJG
files with a different version than the one used for compression. You
may download older versions of packJPG from:
http://www.elektronik.htw-aalen.de/packJPG/binaries/old/
Open source release / developer info
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The packJPG source codes is found inside the "source" subdirectory.
Additional documents aimed to developers, containing detailed
instructions on compiling the source code and using special
functionality, are included in the "packJPG" subdirectory.
History
~~~~~~~
v1.9a (04/20/2007) (non public)
- first released version
- only for testing purposes
v2.0 (05/28/2007) (public)
- first public version of packJPG
- minor improvements to overall compression
- minor bugfixes
v2.2 (08/05/2007) (public)
- around 40% faster compression & decompression
- major improvements to overall compression (around 2% on average)
- reading from stdin, writing to stdout
- smaller executable
- minor bugfixes
- various minor improvements
v2.3 (09/18/2007) (public)
- compatibility with JPEG progressive mode
- compatibility with JPEG extended sequential mode
- compatibility with the CMYK color space
- compatibility with older CPUs
- around 15% faster compression & decompression
- new switch: [-d] (discard meta-info)
- various bugfixes
v2.3a (11/21/2007) (public)
- crash issue with certain images fixed
- compatibility with packJPG v2.3 maintained
v2.3b (12/20/2007) (public)
- some minor errors in the packJPG library fixed
- compatibility with packJPG v2.3 maintained
v2.4 (03/24/2010) (public)
- major improvements (1%...2%) to overall compression
- around 10% faster compression & decompression
- major improvements to JPG compatibility
- size of executable reduced to ~33%
- new switch: [-ver] (verify file after processing)
- new switch: [-np] (no pause after processing)
- new progress bar output mode
- arithmetic coding routines rewritten from scratch
- various smaller improvements to numerous to list here
- new SFX (self extracting) archive format
v2.5 (11/11/2011) (public)
- improvements (~0.5%) to overall compression
- several minor bugfixes
- major code cleanup
- removed packJPX from the package
- added packARC to the package
- packJPG is now open source!
v2.5a (11/21/11) (public)
- source code compatibility improvements (Gerhard Seelmann)
- avoid some compiler warnings (Gerhard Seelmann)
- source code clean up (Gerhard Seelmann)
v2.5b (01/27/12) (public)
- further removal of redundant code
- some fixes for the packJPG static library
- compiler fix for Mac OS (thanks to Sergio Lopez)
- improved compression ratio calculation
- eliminated the need for temp files
v2.5c (04/13/12) (public)
- various source code optimizations
v2.5d (07/03/12) (public)
- fixed a rare bug with progressive JPEG
v2.5e (07/03/12) (public)
- some minor source code optimizations
- changed packJPG licensing to LGPL
- moved packARC to a separate package
v2.5f (02/24/13) (public)
- fixed a minor bug in the JPG parser (thanks to Stephan Busch)
v2.5g (09/14/13) (public)
- fixed a rare crash bug with manipulated JPEG files
Acknowledgements
~~~~~~~~~~~~~~~~
packJPG is the result of countless hours of research and development. It
is part of my final year project for Hochschule Aalen.
Prof. Dr. Gerhard Seelmann from Hochschule Aalen supported my
development of packJPG with his extensive knowledge in the field of data
compression. Without his advice, packJPG would not be possible.
The official homepage of packJPG is currently maintained by Hochschule
Aalen staff.
packJPG logo and icon are designed by Michael Kaufmann.
Contact
~~~~~~~
The official home of packJPG:
http://www.elektronik.htw-aalen.de/packjpg/
For questions and bug reports:
packjpg (at) htw-aalen.de
____________________________________
packJPG by Matthias Stirner, 09/2013

1044
filters/packjpg/aricoder.cpp Normal file
View file

File diff suppressed because it is too large Load diff

268
filters/packjpg/aricoder.h Normal file
View file

@ -0,0 +1,268 @@
// defines for coder
#define CODER_USE_BITS 31 // must never be above 31
#define CODER_LIMIT100 ( (unsigned int) ( 1 << CODER_USE_BITS ) )
#define CODER_LIMIT025 ( ( CODER_LIMIT100 / 4 ) * 1 )
#define CODER_LIMIT050 ( ( CODER_LIMIT100 / 4 ) * 2 )
#define CODER_LIMIT075 ( ( CODER_LIMIT100 / 4 ) * 3 )
#define CODER_MAXSCALE CODER_LIMIT025 - 1
#define ESCAPE_SYMBOL CODER_LIMIT025
// symbol struct, used in arithmetic coding
struct symbol {
unsigned int low_count;
unsigned int high_count;
unsigned int scale;
};
// table struct, used in in statistical models,
// holding all info needed for one context
struct table {
// counts for each symbol contained in the table
unsigned short* counts;
// links to higher order contexts
struct table** links;
// link to lower order context
struct table* lesser;
// accumulated counts
unsigned int scale;
};
// special table struct, used in in model_s,
// holding additional info for a speedier 'totalize_table'
struct table_s {
// counts for each symbol contained in the table
unsigned short* counts;
// links to higher order contexts
struct table_s** links;
// link to lower order context
struct table_s* lesser;
// speedup info
unsigned short max_count;
unsigned short max_symbol;
// unsigned short esc_prob;
};
/* -----------------------------------------------
class for arithmetic coding of data to/from iostream
----------------------------------------------- */
class aricoder
{
public:
aricoder( iostream* stream, int iomode );
~aricoder( void );
void encode( symbol* s );
unsigned int decode_count( symbol* s );
void decode( symbol* s );
private:
// bitwise operations
void write_bit( unsigned char bit );
unsigned char read_bit( void );
// i/o variables
iostream* sptr;
int mode;
unsigned char bbyte;
unsigned char cbit;
// arithmetic coding variables
unsigned int ccode;
unsigned int clow;
unsigned int chigh;
unsigned int cstep;
unsigned int nrbits;
};
/* -----------------------------------------------
universal statistical model for arithmetic coding
----------------------------------------------- */
class model_s
{
public:
model_s( int max_s, int max_c, int max_o, int c_lim );
~model_s( void );
void update_model( int symbol );
void shift_context( int c );
void flush_model( int scale_factor );
void exclude_symbols( char rule, int c );
int convert_int_to_symbol( int c, symbol *s );
void get_symbol_scale( symbol *s );
int convert_symbol_to_int( int count, symbol *s );
bool error;
private:
// unsigned short* totals;
unsigned int* totals;
char* scoreboard;
int sb0_count;
table_s **contexts;
table_s **storage;
int max_symbol;
int max_context;
int current_order;
int max_order;
int max_count;
inline void totalize_table(table_s* context );
inline void rescale_table(table_s* context, int scale_factor );
inline void recursive_flush(table_s* context, int scale_factor );
inline void recursive_cleanup(table_s* context );
};
/* -----------------------------------------------
binary statistical model for arithmetic coding
----------------------------------------------- */
class model_b
{
public:
model_b( int max_c, int max_o, int c_lim );
~model_b( void );
void update_model( int symbol );
void shift_context( int c );
void flush_model( int scale_factor );
int convert_int_to_symbol( int c, symbol *s );
void get_symbol_scale( symbol *s );
int convert_symbol_to_int( int count, symbol *s );
bool error;
private:
table **contexts;
table **storage;
int max_context;
int max_order;
int max_count;
inline void check_counts( table *context );
inline void rescale_table( table* context, int scale_factor );
inline void recursive_flush( table* context, int scale_factor );
inline void recursive_cleanup( table *context );
};
/* -----------------------------------------------
shift context x2 model_s function
----------------------------------------------- */
static inline void shift_model( model_s* model, int ctx1, int ctx2 )
{
model->shift_context( ctx1 );
model->shift_context( ctx2 );
}
/* -----------------------------------------------
shift context x3 model_s function
----------------------------------------------- */
static inline void shift_model( model_s* model, int ctx1, int ctx2, int ctx3 )
{
model->shift_context( ctx1 );
model->shift_context( ctx2 );
model->shift_context( ctx3 );
}
/* -----------------------------------------------
shift context x2 model_b function
----------------------------------------------- */
static inline void shift_model( model_b* model, int ctx1, int ctx2 )
{
model->shift_context( ctx1 );
model->shift_context( ctx2 );
}
/* -----------------------------------------------
shift context x3 model_b function
----------------------------------------------- */
static inline void shift_model( model_b* model, int ctx1, int ctx2, int ctx3 )
{
model->shift_context( ctx1 );
model->shift_context( ctx2 );
model->shift_context( ctx3 );
}
/* -----------------------------------------------
generic model_s encoder function
----------------------------------------------- */
static inline void encode_ari( aricoder* encoder, model_s* model, int c )
{
static symbol s;
static int esc;
do {
esc = model->convert_int_to_symbol( c, &s );
encoder->encode( &s );
} while ( esc );
model->update_model( c );
}
/* -----------------------------------------------
generic model_s decoder function
----------------------------------------------- */
static inline int decode_ari( aricoder* decoder, model_s* model )
{
static symbol s;
static unsigned int count;
static int c;
do{
model->get_symbol_scale( &s );
count = decoder->decode_count( &s );
c = model->convert_symbol_to_int( count, &s );
decoder->decode( &s );
} while ( c == ESCAPE_SYMBOL );
model->update_model( c );
return c;
}
/* -----------------------------------------------
generic model_b encoder function
----------------------------------------------- */
static inline void encode_ari( aricoder* encoder, model_b* model, int c )
{
static symbol s;
model->convert_int_to_symbol( c, &s );
encoder->encode( &s );
model->update_model( c );
}
/* -----------------------------------------------
generic model_b decoder function
----------------------------------------------- */
static inline int decode_ari( aricoder* decoder, model_b* model )
{
static symbol s;
static unsigned int count;
static int c;
model->get_symbol_scale( &s );
count = decoder->decode_count( &s );
c = model->convert_symbol_to_int( count, &s );
decoder->decode( &s );
model->update_model( c );
return c;
}

910
filters/packjpg/bitops.cpp Normal file
View file

@ -0,0 +1,910 @@
/*
This file contains special classes for bitwise
reading and writing of arrays
*/
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include "bitops.h"
#define BUFFER_SIZE 1024 * 1024
/* -----------------------------------------------
constructor for abitreader class
----------------------------------------------- */
abitreader::abitreader( unsigned char* array, int size )
{
cbyte = 0;
cbit = 8;
peof = 0;
eof = false;
data = array;
lbyte = size;
}
/* -----------------------------------------------
destructor for abitreader class
----------------------------------------------- */
abitreader::~abitreader( void )
{
}
/* -----------------------------------------------
reads n bits from abitreader
----------------------------------------------- */
unsigned int abitreader::read( int nbits )
{
unsigned int retval = 0;
// safety check for eof
if ( eof ) {
peof += nbits;
return 0;
}
while ( nbits >= cbit ) {
nbits -= cbit;
retval |= ( RBITS( data[cbyte], cbit ) << nbits );
cbit = 8;
if ( ++cbyte >= lbyte ) {
peof = nbits;
eof = true;
return retval;
}
}
if ( nbits > 0 ) {
retval |= ( MBITS( data[cbyte], cbit, (cbit-nbits) ) );
cbit -= nbits;
}
return retval;
}
/* -----------------------------------------------
reads one bit from abitreader
----------------------------------------------- */
unsigned char abitreader::read_bit( void )
{
unsigned char bit;
// safety check for eof
if (eof) {
peof++;
return 0;
}
// read one bit
bit = BITN( data[cbyte], --cbit );
if ( cbit == 0 ) {
if ( ++cbyte == lbyte ) eof = true;
cbit = 8;
}
return bit;
}
/* -----------------------------------------------
to skip padding from current byte
----------------------------------------------- */
unsigned char abitreader::unpad( unsigned char fillbit )
{
if ( ( cbit == 8 ) || eof ) return fillbit;
else {
fillbit = read( 1 );
while ( cbit != 8 ) read( 1 );
}
return fillbit;
}
/* -----------------------------------------------
get current position in array
----------------------------------------------- */
int abitreader::getpos( void )
{
return cbyte;
}
/* -----------------------------------------------
get current bit position
----------------------------------------------- */
int abitreader::getbitp( void )
{
return cbit;
}
/* -----------------------------------------------
set byte and bit position
----------------------------------------------- */
void abitreader::setpos( int pbyte, int pbit )
{
if ( pbyte < lbyte ) {
// reset eof
eof = false;
// set positions
cbyte = pbyte;
cbit = pbit;
} else {
// set eof
eof = true;
// set positions
cbyte = lbyte;
cbit = 8;
peof = ( ( pbyte - lbyte ) * 8 ) + 8 - pbit;
}
}
/* -----------------------------------------------
rewind n bits
----------------------------------------------- */
void abitreader::rewind_bits( int nbits )
{
if ( eof ) {
if ( nbits > peof ) nbits -= peof;
else {
peof -= nbits;
return;
}
eof = false;
}
for ( cbit += nbits; cbit > 8; cbyte--, cbit -= 8 );
if ( cbyte < 0 ) {
cbyte = 0;
cbit = 8;
}
}
/* -----------------------------------------------
constructor for abitwriter class
----------------------------------------------- */
abitwriter::abitwriter( int size )
{
fillbit = 1;
adds = 65536;
cbyte = 0;
cbit = 8;
error = false;
fmem = true;
dsize = ( size > 0 ) ? size : adds;
data = ( unsigned char* ) malloc ( dsize );
if ( data == NULL ) {
error = true;
return;
}
// fill buffer with zeroes
memset( data, 0, dsize * sizeof( char ) );
// for ( int i = 0; i < dsize; i++ ) data[i] = 0;
}
/* -----------------------------------------------
destructor for abitwriter class
----------------------------------------------- */
abitwriter::~abitwriter( void )
{
// free memory if pointer was not given out
if ( fmem ) free( data );
}
/* -----------------------------------------------
writes n bits to abitwriter
----------------------------------------------- */
void abitwriter::write( unsigned int val, int nbits )
{
// safety check for error
if ( error ) return;
// test if pointer beyond flush treshold
if ( cbyte > ( dsize - 5 ) ) {
dsize += adds;
data = (unsigned char*) realloc( data, dsize );
if ( data == NULL ) {
error = true;
return;
}
memset( ( data + cbyte + 1 ), 0, ( dsize - ( cbyte + 1 ) ) * sizeof( char ) );
}
// write data
while ( nbits >= cbit ) {
data[cbyte] |= ( MBITS32(val, nbits, (nbits-cbit)) );
nbits -= cbit;
cbyte++;
cbit = 8;
}
if ( nbits > 0 ) {
data[cbyte] |= ( (RBITS32(val, nbits)) << (cbit - nbits) );
cbit -= nbits;
}
}
/* -----------------------------------------------
writes one bit to abitwriter
----------------------------------------------- */
void abitwriter::write_bit( unsigned char bit )
{
// safety check for error
if ( error ) return;
// write data
if ( bit ) data[cbyte] |= 0x1 << (--cbit);
else --cbit;
if ( cbit == 0 ) {
// test if pointer beyond flush treshold
if ( ++cbyte > ( dsize - 5 ) ) {
dsize += adds;
data = (unsigned char*) realloc( data, dsize );
if ( data == NULL ) {
error = true;
return;
}
memset( ( data + cbyte + 1 ), 0, ( dsize - ( cbyte + 1 ) ) * sizeof( char ) );
}
cbit = 8;
}
}
/* -----------------------------------------------
pads data using fillbit
----------------------------------------------- */
void abitwriter::pad( unsigned char fillbit )
{
while ( cbit < 8 )
write( fillbit, 1 );
}
/* -----------------------------------------------
gets data array from abitwriter
----------------------------------------------- */
unsigned char* abitwriter::getptr( void )
{
// data is padded here
pad( fillbit );
// forbid freeing memory
fmem = false;
// realloc data
data = (unsigned char*) realloc( data, cbyte );
return data;
}
/* -----------------------------------------------
gets size of data array from abitwriter
----------------------------------------------- */
int abitwriter::getpos( void )
{
return cbyte;
}
/* -----------------------------------------------
get current bit position
----------------------------------------------- */
int abitwriter::getbitp( void )
{
return cbit;
}
/* -----------------------------------------------
constructor for abytewriter class
----------------------------------------------- */
abytereader::abytereader( unsigned char* array, int size )
{
cbyte = 0;
eof = false;
data = array;
lbyte = size;
if ( ( data == NULL ) || ( lbyte == 0 ) )
eof = true;
}
/* -----------------------------------------------
destructor for abytewriter class
----------------------------------------------- */
abytereader::~abytereader( void )
{
}
/* -----------------------------------------------
reads 1 byte from abytereader
----------------------------------------------- */
int abytereader::read( unsigned char* byte )
{
if ( cbyte >= lbyte ) {
cbyte = lbyte;
eof = true;
return 0;
}
else {
*byte = data[ cbyte++ ];
return 1;
}
}
/* -----------------------------------------------
reads n bytes from abytereader
----------------------------------------------- */
int abytereader::read_n( unsigned char* byte, int n )
{
int nl = lbyte - cbyte;
int i;
if ( nl < n ) {
for ( i = 0; i < nl; i++ )
byte[ i ] = data[ cbyte + i ];
cbyte = lbyte;
eof = true;
return nl;
}
else {
for ( i = 0; i < n; i++ )
byte[ i ] = data[ cbyte + i ];
cbyte += n;
return n;
}
}
/* -----------------------------------------------
go to position in data
----------------------------------------------- */
void abytereader::seek( int pos )
{
if ( pos >= lbyte ) {
cbyte = lbyte;
eof = true;
}
else {
cbyte = pos;
eof = false;
}
}
/* -----------------------------------------------
gets size of current data
----------------------------------------------- */
int abytereader::getsize( void )
{
return lbyte;
}
/* -----------------------------------------------
gets current position from abytereader
----------------------------------------------- */
int abytereader::getpos( void )
{
return cbyte;
}
/* -----------------------------------------------
constructor for abytewriter class
----------------------------------------------- */
abytewriter::abytewriter( int size )
{
adds = 65536;
cbyte = 0;
error = false;
fmem = true;
dsize = ( size > 0 ) ? size : adds;
data = (unsigned char*) malloc( dsize );
if ( data == NULL ) {
error = true;
return;
}
}
/* -----------------------------------------------
destructor for abytewriter class
----------------------------------------------- */
abytewriter::~abytewriter( void )
{
// free data if pointer is not read
if ( fmem ) free( data );
}
/* -----------------------------------------------
writes 1 byte to abytewriter
----------------------------------------------- */
void abytewriter::write( unsigned char byte )
{
// safety check for error
if ( error ) return;
// test if pointer beyond flush threshold
if ( cbyte >= ( dsize - 2 ) ) {
dsize += adds;
data = (unsigned char*) realloc( data, dsize );
if ( data == NULL ) {
error = true;
return;
}
}
// write data
data[ cbyte++ ] = byte;
}
/* -----------------------------------------------
writes n byte to abytewriter
----------------------------------------------- */
void abytewriter::write_n( unsigned char* byte, int n )
{
// safety check for error
if ( error ) return;
// make sure that pointer doesn't get beyond flush threshold
while ( ( cbyte + n ) >= ( dsize - 2 ) ) {
dsize += adds;
data = (unsigned char*) realloc( data, dsize );
if ( data == NULL ) {
error = true;
return;
}
}
// copy data from array
while ( n-- > 0 )
data[ cbyte++ ] = *(byte++);
}
/* -----------------------------------------------
gets data array from abytewriter
----------------------------------------------- */
unsigned char* abytewriter::getptr( void )
{
// safety check for error
if ( error ) return NULL;
// forbid freeing memory
fmem = false;
// realloc data
data = (unsigned char*) realloc( data, cbyte );
return data;
}
/* -----------------------------------------------
peeks into data array from abytewriter
----------------------------------------------- */
unsigned char* abytewriter::peekptr( void )
{
return data;
}
/* -----------------------------------------------
gets size of data array from abytewriter
----------------------------------------------- */
int abytewriter::getpos( void )
{
return cbyte;
}
/* -----------------------------------------------
reset without realloc
----------------------------------------------- */
void abytewriter::reset( void )
{
// set position of current byte
cbyte = 0;
}
/* -----------------------------------------------
constructor for iostream class
----------------------------------------------- */
iostream::iostream( void* src, int srctype, int srcsize, int iomode )
{
// locally copy source, source type # and io mode #
source = src;
srct = srctype;
srcs = srcsize;
mode = iomode;
// don't free memory when reading - this will be useful if switching occurs
free_mem_sw = false;
// set binary mode for streams
#if defined( _WIN32 )
setmode( fileno( stdin ), O_BINARY );
setmode( fileno( stdout ), O_BINARY );
#endif
// open file/mem/stream
switch ( srct )
{
case 0:
open_file();
break;
case 1:
open_mem();
break;
case 2:
open_stream();
break;
default:
break;
}
}
/* -----------------------------------------------
destructor for iostream class
----------------------------------------------- */
iostream::~iostream( void )
{
// if needed, write memory to stream or free memory from buffered stream
if ( srct == 2 ) {
if ( mode == 1 ) {
if ( !(mwrt->error) ) {
srcs = mwrt->getpos();
source = mwrt->getptr();
fwrite( source, sizeof( char ), srcs, stdout );
}
}
}
// free all buffers
if ( srct == 0 ) {
if ( fptr != NULL )
fclose( fptr );
}
else if ( mode == 0 ) {
if ( free_mem_sw )
free( source );
delete( mrdr );
}
else
delete( mwrt );
}
/* -----------------------------------------------
switches mode from reading to writing and vice versa
----------------------------------------------- */
void iostream::switch_mode( void )
{
// return immediately if there's an error
if ( chkerr() ) return;
if ( mode == 0 ) {
// WARNING: when switching from reading to writing, information might be lost forever
switch ( srct ) {
case 0:
fclose( fptr );
fptr = fopen( ( char* ) source, "wb" );
break;
case 1:
case 2:
delete( mrdr );
if ( free_mem_sw )
free( source ); // see? I've told you so :-)
mwrt = new abytewriter( srcs );
break;
default:
break;
}
mode = 1;
}
else {
// switching from writing to reading is a bit more complicated
switch ( srct ) {
case 0:
fclose( fptr );
fptr = fopen( ( char* ) source, "rb" );
break;
case 1:
case 2:
source = mwrt->getptr();
srcs = mwrt->getpos();
delete( mwrt );
mrdr = new abytereader( ( unsigned char* ) source, srcs );
free_mem_sw = true;
break;
default:
break;
}
mode = 0;
}
}
/* -----------------------------------------------
generic read function
----------------------------------------------- */
int iostream::read( void* to, int tpsize, int dtsize )
{
return ( srct == 0 ) ? read_file( to, tpsize, dtsize ) : read_mem( to, tpsize, dtsize );
}
/* -----------------------------------------------
generic write function
----------------------------------------------- */
int iostream::write( void* from, int tpsize, int dtsize )
{
return ( srct == 0 ) ? write_file( from, tpsize, dtsize ) : write_mem( from, tpsize, dtsize );
}
/* -----------------------------------------------
flush function
----------------------------------------------- */
int iostream::flush( void )
{
if ( srct == 0 )
fflush( fptr );
return getpos();
}
/* -----------------------------------------------
rewind to beginning of stream
----------------------------------------------- */
int iostream::rewind( void )
{
// WARNING: when writing, rewind might lose all your data
if ( srct == 0 )
fseek( fptr, 0, SEEK_SET );
else if ( mode == 0 )
mrdr->seek( 0 );
else
mwrt->reset();
return getpos();
}
/* -----------------------------------------------
get current position in stream
----------------------------------------------- */
int iostream::getpos( void )
{
int pos;
if ( srct == 0 )
pos = ftell( fptr );
else if ( mode == 0 )
pos = mrdr->getpos();
else
pos = mwrt->getpos();
return pos;
}
/* -----------------------------------------------
get size of file
----------------------------------------------- */
int iostream::getsize( void )
{
int pos;
int siz;
if ( mode == 0 ) {
if ( srct == 0 ) {
pos = ftell( fptr );
fseek( fptr, 0, SEEK_END );
siz = ftell( fptr );
fseek( fptr, pos, SEEK_SET );
}
else {
siz = mrdr->getsize();
}
}
else {
siz = getpos();
}
return siz;
}
/* -----------------------------------------------
get data pointer (for mem io only)
----------------------------------------------- */
unsigned char* iostream::getptr( void )
{
if ( srct == 1 )
return ( mode == 0 ) ? ( unsigned char* ) source : mwrt->getptr();
else
return NULL;
}
/* -----------------------------------------------
check for errors
----------------------------------------------- */
bool iostream::chkerr( void )
{
bool error = false;
// check for user input errors
if ( ( mode != 0 ) && ( mode != 1 ) )
error = true;
if ( ( srct != 0 ) && ( srct != 1 ) && ( srct != 2 ) )
error = true;
// check for io errors
if ( srct == 0 ) {
if ( fptr == NULL )
error = true;
else if ( ferror( fptr ) )
error = true;
}
else if ( mode == 0 ) {
if ( mrdr == NULL )
error = true;
}
else {
if ( mwrt == NULL )
error = true;
else if ( mwrt->error )
error = true;
}
return error;
}
/* -----------------------------------------------
check for eof (read only)
----------------------------------------------- */
bool iostream::chkeof( void )
{
if ( mode == 0 )
return ( srct == 0 ) ? feof( fptr ) : mrdr->eof;
else
return false;
}
/* -----------------------------------------------
open function for files
----------------------------------------------- */
void iostream::open_file( void )
{
char* fn = (char*) source;
// open file for reading / writing
fptr = fopen( fn, ( mode == 0 ) ? "rb" : "wb" );
}
/* -----------------------------------------------
open function for memory
----------------------------------------------- */
void iostream::open_mem( void )
{
if ( mode == 0 )
mrdr = new abytereader( ( unsigned char* ) source, srcs );
else
mwrt = new abytewriter( srcs );
}
/* -----------------------------------------------
open function for streams
----------------------------------------------- */
void iostream::open_stream( void )
{
abytewriter* strwrt;
unsigned char* buffer;
int i;
if ( mode == 0 ) {
// read whole stream into memory buffer
strwrt = new abytewriter( 0 );
buffer = ( unsigned char* ) calloc( BUFFER_SIZE, sizeof( char ) );
if ( buffer != NULL ) {
while ( ( i = fread( buffer, sizeof( char ), BUFFER_SIZE, stdin ) ) > 0 )
strwrt->write_n( buffer, i );
}
if ( strwrt->error ) {
source = NULL;
srcs = 0;
}
else {
source = strwrt->getptr();
srcs = strwrt->getpos();
}
delete ( strwrt );
free( buffer );
// free memory after done
free_mem_sw = true;
}
// for writing: simply open new stream in mem writer
// writing to stream will be done later
open_mem();
}
/* -----------------------------------------------
write function for files
----------------------------------------------- */
int iostream::write_file( void* from, int tpsize, int dtsize )
{
return fwrite( from, tpsize, dtsize, fptr );
}
/* -----------------------------------------------
read function for files
----------------------------------------------- */
int iostream::read_file( void* to, int tpsize, int dtsize )
{
return fread( to, tpsize, dtsize, fptr );
}
/* -----------------------------------------------
write function for memory
----------------------------------------------- */
int iostream::write_mem( void* from, int tpsize, int dtsize )
{
int n = tpsize * dtsize;
mwrt->write_n( ( unsigned char* ) from, n );
return ( mwrt->error ) ? 0 : n;
}
/* -----------------------------------------------
read function for memory
----------------------------------------------- */
int iostream::read_mem( void* to, int tpsize, int dtsize )
{
int n = tpsize * dtsize;
return ( mrdr->read_n( ( unsigned char* ) to, n ) ) / tpsize;
}

169
filters/packjpg/bitops.h Normal file
View file

@ -0,0 +1,169 @@
#define RBITS( c, n ) ( c & ( 0xFF >> (8 - n) ) )
#define LBITS( c, n ) ( c >> (8 - n) )
#define MBITS( c, l, r ) ( RBITS( c,l ) >> r )
#define RBITS16( c, n ) ( c & ( 0xFFFFFFFF >> (16 - n) ) )
#define LBITS16( c, n ) ( c >> (16 - n) )
#define MBITS16( c, l, r ) ( RBITS16( c,l ) >> r )
#define RBITS32( c, n ) ( c & ( 0xFFFFFFFF >> (32 - n) ) )
#define LBITS32( c, n ) ( c >> (32 - n) )
#define MBITS32( c, l, r ) ( RBITS32( c,l ) >> r )
#define BITN( c, n ) ( (c >> n) & 0x1 )
#define BITLEN( l, v ) for ( l = 0; ( v >> l ) > 0; l++ )
#define FDIV2( v, p ) ( ( v < 0 ) ? -( (-v) >> p ) : ( v >> p ) )
#define TYPE_FILE 0
#define TYPE_MEMORY 1
#define TYPE_STREAM 2
#define MODE_READ 0
#define MODE_WRITE 1
#define BTST_BUFF 1024 * 1024
#include <stdio.h>
/* -----------------------------------------------
class to read arrays bitwise
----------------------------------------------- */
class abitreader
{
public:
abitreader( unsigned char* array, int size );
~abitreader( void );
unsigned int read( int nbits );
unsigned char read_bit( void );
unsigned char unpad( unsigned char fillbit );
int getpos( void );
int getbitp( void );
void setpos( int pbyte, int pbit );
void rewind_bits( int nbits );
bool eof;
int peof;
private:
unsigned char* data;
int lbyte;
int cbyte;
int cbit;
};
/* -----------------------------------------------
class to write arrays bitwise
----------------------------------------------- */
class abitwriter
{
public:
abitwriter( int size );
~abitwriter( void );
void write( unsigned int val, int nbits );
void write_bit( unsigned char bit );
void pad ( unsigned char fillbit );
unsigned char* getptr( void );
int getpos( void );
int getbitp( void );
bool error;
unsigned char fillbit;
private:
unsigned char* data;
int dsize;
int adds;
int lbyte;
int cbyte;
int cbit;
bool fmem;
};
/* -----------------------------------------------
class to read arrays bytewise
----------------------------------------------- */
class abytereader
{
public:
abytereader( unsigned char* array, int size );
~abytereader( void );
int read( unsigned char* byte );
int read_n( unsigned char* byte, int n );
void seek( int pos );
int getsize( void );
int getpos( void );
bool eof;
private:
unsigned char* data;
int lbyte;
int cbyte;
};
/* -----------------------------------------------
class to write arrays bytewise
----------------------------------------------- */
class abytewriter
{
public:
abytewriter( int size );
~abytewriter( void );
void write( unsigned char byte );
void write_n( unsigned char* byte, int n );
unsigned char* getptr( void );
unsigned char* peekptr( void );
int getpos( void );
void reset( void );
bool error;
private:
unsigned char* data;
int dsize;
int adds;
int lbyte;
int cbyte;
bool fmem;
};
/* -----------------------------------------------
class for input and output from file or memory
----------------------------------------------- */
class iostream
{
public:
iostream( void* src, int srctype, int srcsize, int iomode );
~iostream( void );
void switch_mode( void );
int read( void* to, int tpsize, int dtsize );
int write( void* from, int tpsize, int dtsize );
int flush( void );
int rewind( void );
int getpos( void );
int getsize( void );
unsigned char* getptr( void );
bool chkerr( void );
bool chkeof( void );
private:
void open_file( void );
void open_mem( void );
void open_stream( void );
int write_file( void* from, int tpsize, int dtsize );
int read_file( void* to, int tpsize, int dtsize );
int write_mem( void* from, int tpsize, int dtsize );
int read_mem( void* to, int tpsize, int dtsize );
FILE* fptr;
abytewriter* mwrt;
abytereader* mrdr;
bool free_mem_sw;
void* source;
int mode;
int srct;
int srcs;
};

1166
filters/packjpg/dct8x8.h Normal file
View file

File diff suppressed because it is too large Load diff

165
filters/packjpg/lgpl-3.0.txt Normal file
View file

@ -0,0 +1,165 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.

7213
filters/packjpg/packjpg.cpp Normal file
View file

File diff suppressed because it is too large Load diff

40
filters/packjpg/packjpglib.h Normal file
View file

@ -0,0 +1,40 @@
// packJPGlib.h - function declarations for the packJPG library
#if defined BUILD_DLL
#define EXPORT __declspec( dllexport )
#else
#define EXPORT extern
#endif
/* -----------------------------------------------
function declarations: library only functions
----------------------------------------------- */
EXPORT bool pjglib_convert_stream2stream( char* msg );
EXPORT bool pjglib_convert_file2file( char* in, char* out, char* msg );
EXPORT bool pjglib_convert_stream2mem( unsigned char** out_file, unsigned int* out_size, char* msg );
EXPORT void pjglib_init_streams( void* in_src, int in_type, int in_size, void* out_dest, int out_type );
EXPORT const char* pjglib_version_info( void );
EXPORT const char* pjglib_short_name( void );
/* a short reminder about input/output stream types
for the pjglib_init_streams() function
if input is file
----------------
in_scr -> name of input file
in_type -> 0
in_size -> ignore
if input is memory
------------------
in_scr -> array containg data
in_type -> 1
in_size -> size of data array
if input is *FILE (f.e. stdin)
------------------------------
in_src -> stream pointer
in_type -> 2
in_size -> ignore
vice versa for output streams! */

1659
filters/packjpg/pjpgtbl.h Normal file
View file

File diff suppressed because it is too large Load diff

0
transpose/transpose.c → filters/transpose/transpose.c
View file

0
transpose/transpose.h → filters/transpose/transpose.h
View file