From dc867e363b3609019e0e4985c8a4a9d65f16f1eb Mon Sep 17 00:00:00 2001 From: Jon Meredith Date: Thu, 4 Jun 2009 14:54:27 -0600 Subject: [PATCH] Incorporated Diz's crc generation/checking patch to bdberl.erl. Added crc checking before/after BDB accesses in C driver. Changed get to use malloc DBTs rather than the user supplied buffer previously. --- c_src/bdberl_crc32.c | 113 +++++++++++++++++++++++++++++++++++++++++++ c_src/bdberl_crc32.h | 14 ++++++ c_src/bdberl_drv.c | 89 ++++++++++++++++++++++++++++++---- c_src/bdberl_drv.h | 3 +- src/bdberl.erl | 28 +++++++++-- 5 files changed, 232 insertions(+), 15 deletions(-) create mode 100644 c_src/bdberl_crc32.c create mode 100644 c_src/bdberl_crc32.h diff --git a/c_src/bdberl_crc32.c b/c_src/bdberl_crc32.c new file mode 100644 index 0000000..d20c846 --- /dev/null +++ b/c_src/bdberl_crc32.c @@ -0,0 +1,113 @@ +/* ------------------------------------------------------------------- + * + * bdberl: CRC-32 checks + * Copyright (c) 2008-9 The Hive. All rights reserved. + * + * ------------------------------------------------------------------- */ + +#include "bdberl_crc32.h" + +/*****************************************************************/ +/* */ +/* CRC LOOKUP TABLE */ +/* ================ */ +/* The following CRC lookup table was generated automagically */ +/* by the Rocksoft^tm Model CRC Algorithm Table Generation */ +/* Program V1.0 using the following model parameters: */ +/* */ +/* Width : 4 bytes. */ +/* Poly : 0x04C11DB7L */ +/* Reverse : TRUE. */ +/* */ +/* For more information on the Rocksoft^tm Model CRC Algorithm, */ +/* see the document titled "A Painless Guide to CRC Error */ +/* Detection Algorithms" by Ross Williams */ +/* (ross@guest.adelaide.edu.au.). This document is likely to be */ +/* in the FTP archive "ftp.adelaide.edu.au/pub/rocksoft". */ +/* */ +/*****************************************************************/ + +static uint32_t crctable[256] = +{ + 0x00000000L, 0x77073096L, 0xEE0E612CL, 0x990951BAL, + 0x076DC419L, 0x706AF48FL, 0xE963A535L, 0x9E6495A3L, + 0x0EDB8832L, 0x79DCB8A4L, 0xE0D5E91EL, 0x97D2D988L, + 0x09B64C2BL, 0x7EB17CBDL, 0xE7B82D07L, 0x90BF1D91L, + 0x1DB71064L, 0x6AB020F2L, 0xF3B97148L, 0x84BE41DEL, + 0x1ADAD47DL, 0x6DDDE4EBL, 0xF4D4B551L, 0x83D385C7L, + 0x136C9856L, 0x646BA8C0L, 0xFD62F97AL, 0x8A65C9ECL, + 0x14015C4FL, 0x63066CD9L, 0xFA0F3D63L, 0x8D080DF5L, + 0x3B6E20C8L, 0x4C69105EL, 0xD56041E4L, 0xA2677172L, + 0x3C03E4D1L, 0x4B04D447L, 0xD20D85FDL, 0xA50AB56BL, + 0x35B5A8FAL, 0x42B2986CL, 0xDBBBC9D6L, 0xACBCF940L, + 0x32D86CE3L, 0x45DF5C75L, 0xDCD60DCFL, 0xABD13D59L, + 0x26D930ACL, 0x51DE003AL, 0xC8D75180L, 0xBFD06116L, + 0x21B4F4B5L, 0x56B3C423L, 0xCFBA9599L, 0xB8BDA50FL, + 0x2802B89EL, 0x5F058808L, 0xC60CD9B2L, 0xB10BE924L, + 0x2F6F7C87L, 0x58684C11L, 0xC1611DABL, 0xB6662D3DL, + 0x76DC4190L, 0x01DB7106L, 0x98D220BCL, 0xEFD5102AL, + 0x71B18589L, 0x06B6B51FL, 0x9FBFE4A5L, 0xE8B8D433L, + 0x7807C9A2L, 0x0F00F934L, 0x9609A88EL, 0xE10E9818L, + 0x7F6A0DBBL, 0x086D3D2DL, 0x91646C97L, 0xE6635C01L, + 0x6B6B51F4L, 0x1C6C6162L, 0x856530D8L, 0xF262004EL, + 0x6C0695EDL, 0x1B01A57BL, 0x8208F4C1L, 0xF50FC457L, + 0x65B0D9C6L, 0x12B7E950L, 0x8BBEB8EAL, 0xFCB9887CL, + 0x62DD1DDFL, 0x15DA2D49L, 0x8CD37CF3L, 0xFBD44C65L, + 0x4DB26158L, 0x3AB551CEL, 0xA3BC0074L, 0xD4BB30E2L, + 0x4ADFA541L, 0x3DD895D7L, 0xA4D1C46DL, 0xD3D6F4FBL, + 0x4369E96AL, 0x346ED9FCL, 0xAD678846L, 0xDA60B8D0L, + 0x44042D73L, 0x33031DE5L, 0xAA0A4C5FL, 0xDD0D7CC9L, + 0x5005713CL, 0x270241AAL, 0xBE0B1010L, 0xC90C2086L, + 0x5768B525L, 0x206F85B3L, 0xB966D409L, 0xCE61E49FL, + 0x5EDEF90EL, 0x29D9C998L, 0xB0D09822L, 0xC7D7A8B4L, + 0x59B33D17L, 0x2EB40D81L, 0xB7BD5C3BL, 0xC0BA6CADL, + 0xEDB88320L, 0x9ABFB3B6L, 0x03B6E20CL, 0x74B1D29AL, + 0xEAD54739L, 0x9DD277AFL, 0x04DB2615L, 0x73DC1683L, + 0xE3630B12L, 0x94643B84L, 0x0D6D6A3EL, 0x7A6A5AA8L, + 0xE40ECF0BL, 0x9309FF9DL, 0x0A00AE27L, 0x7D079EB1L, + 0xF00F9344L, 0x8708A3D2L, 0x1E01F268L, 0x6906C2FEL, + 0xF762575DL, 0x806567CBL, 0x196C3671L, 0x6E6B06E7L, + 0xFED41B76L, 0x89D32BE0L, 0x10DA7A5AL, 0x67DD4ACCL, + 0xF9B9DF6FL, 0x8EBEEFF9L, 0x17B7BE43L, 0x60B08ED5L, + 0xD6D6A3E8L, 0xA1D1937EL, 0x38D8C2C4L, 0x4FDFF252L, + 0xD1BB67F1L, 0xA6BC5767L, 0x3FB506DDL, 0x48B2364BL, + 0xD80D2BDAL, 0xAF0A1B4CL, 0x36034AF6L, 0x41047A60L, + 0xDF60EFC3L, 0xA867DF55L, 0x316E8EEFL, 0x4669BE79L, + 0xCB61B38CL, 0xBC66831AL, 0x256FD2A0L, 0x5268E236L, + 0xCC0C7795L, 0xBB0B4703L, 0x220216B9L, 0x5505262FL, + 0xC5BA3BBEL, 0xB2BD0B28L, 0x2BB45A92L, 0x5CB36A04L, + 0xC2D7FFA7L, 0xB5D0CF31L, 0x2CD99E8BL, 0x5BDEAE1DL, + 0x9B64C2B0L, 0xEC63F226L, 0x756AA39CL, 0x026D930AL, + 0x9C0906A9L, 0xEB0E363FL, 0x72076785L, 0x05005713L, + 0x95BF4A82L, 0xE2B87A14L, 0x7BB12BAEL, 0x0CB61B38L, + 0x92D28E9BL, 0xE5D5BE0DL, 0x7CDCEFB7L, 0x0BDBDF21L, + 0x86D3D2D4L, 0xF1D4E242L, 0x68DDB3F8L, 0x1FDA836EL, + 0x81BE16CDL, 0xF6B9265BL, 0x6FB077E1L, 0x18B74777L, + 0x88085AE6L, 0xFF0F6A70L, 0x66063BCAL, 0x11010B5CL, + 0x8F659EFFL, 0xF862AE69L, 0x616BFFD3L, 0x166CCF45L, + 0xA00AE278L, 0xD70DD2EEL, 0x4E048354L, 0x3903B3C2L, + 0xA7672661L, 0xD06016F7L, 0x4969474DL, 0x3E6E77DBL, + 0xAED16A4AL, 0xD9D65ADCL, 0x40DF0B66L, 0x37D83BF0L, + 0xA9BCAE53L, 0xDEBB9EC5L, 0x47B2CF7FL, 0x30B5FFE9L, + 0xBDBDF21CL, 0xCABAC28AL, 0x53B39330L, 0x24B4A3A6L, + 0xBAD03605L, 0xCDD70693L, 0x54DE5729L, 0x23D967BFL, + 0xB3667A2EL, 0xC4614AB8L, 0x5D681B02L, 0x2A6F2B94L, + 0xB40BBE37L, 0xC30C8EA1L, 0x5A05DF1BL, 0x2D02EF8DL +}; + +/*****************************************************************/ +/* End of CRC Lookup Table */ +/*****************************************************************/ + +#define INIT_REFLECTED 0xFFFFFFFF +#define XOROT 0xFFFFFFFF + +// Here is the reflected form: +uint32_t bdberl_crc32(const unsigned char *blk_adr, unsigned int blk_len) +{ + + uint32_t crc = INIT_REFLECTED; + while (blk_len--) + crc = crctable[(crc ^ *blk_adr++) & 0xFFL] ^ (crc >> 8); + return crc ^ XOROT; +} diff --git a/c_src/bdberl_crc32.h b/c_src/bdberl_crc32.h new file mode 100644 index 0000000..724dde2 --- /dev/null +++ b/c_src/bdberl_crc32.h @@ -0,0 +1,14 @@ +/* ------------------------------------------------------------------- + * + * bdberl: CRC checks + * Copyright (c) 2008-9 The Hive. All rights reserved. + * + * ------------------------------------------------------------------- */ +#ifndef _BDBERL_CRC32 +#define _BDBERL_CRC32 + +#include + +uint32_t bdberl_crc32(const unsigned char *blk_adr, unsigned int blk_len); + +#endif // _BDBERL_CRC32 diff --git a/c_src/bdberl_drv.c b/c_src/bdberl_drv.c index 6f89628..da75c8e 100644 --- a/c_src/bdberl_drv.c +++ b/c_src/bdberl_drv.c @@ -1424,12 +1424,17 @@ static char *rc_to_atom_str(int rc) switch (rc) { // bdberl driver errors + case ERROR_MAX_DBS: return "max_dbs"; case ERROR_ASYNC_PENDING: return "async_pending"; case ERROR_INVALID_DBREF: return "invalid_db"; case ERROR_TXN_OPEN: return "transaction_open"; case ERROR_NO_TXN: return "no_txn"; case ERROR_CURSOR_OPEN: return "cursor_open"; case ERROR_NO_CURSOR: return "no_cursor"; + case ERROR_DB_ACTIVE: return "db_active"; + case ERROR_INVALID_CMD: return "invalid_cmd"; + case ERROR_INVALID_DB_TYPE: return "invalid_db_type"; + case ERROR_INVALID_VALUE: return "invalid_value"; // bonafide BDB errors case DB_BUFFER_SMALL: return "buffer_small"; case DB_DONOTINDEX: return "do_not_index"; @@ -1573,10 +1578,24 @@ static void async_cleanup_and_send_kv(PortData* d, int rc, DBT* key, DBT* value) } else { - ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("error"), - ERL_DRV_INT, rc, - ERL_DRV_TUPLE, 2}; - driver_send_term(port, pid, response, sizeof(response) / sizeof(response[0])); + // See if this is a standard errno that we have an erlang code for + char *error = rc_to_atom_str(rc); + if (NULL != error) + { + ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("error"), + ERL_DRV_ATOM, driver_mk_atom(error), + ERL_DRV_TUPLE, 2}; + driver_send_term(port, pid, response, sizeof(response) / sizeof(response[0])); + } + else + { + ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("error"), + ERL_DRV_ATOM, driver_mk_atom("unknown"), + ERL_DRV_INT, rc, + ERL_DRV_TUPLE, 2, + ERL_DRV_TUPLE, 2}; + driver_send_term(port, pid, response, sizeof(response) / sizeof(response[0])); + } } } @@ -2121,10 +2140,23 @@ static void do_async_put(void* arg) value.size = UNPACK_INT(d->work_buffer, 12 + key.size); value.data = UNPACK_BLOB(d->work_buffer, 12 + key.size + 4); - // Execute the actual put. All databases are opened with AUTO_COMMIT, so if msg->port->txn - // is NULL, the put will still be atomic - int rc = db->put(db, d->txn, &key, &value, flags); + // Check CRC in value payload - first 4 bytes are CRC of rest of bytes + assert(value.size >= 4); + uint32_t calc_crc32 = bdberl_crc32(value.data+4, value.size-4); + uint32_t file_crc32 = *(uint32_t*) value.data; + int rc; + if (calc_crc32 != file_crc32) + { + rc = ERROR_INVALID_VALUE; + } + else + { + // Execute the actual put. All databases are opened with AUTO_COMMIT, so if msg->port->txn + // is NULL, the put will still be atomic + rc = db->put(db, d->txn, &key, &value, flags); + } + // If any error occurs while we have a txn action, abort it if (d->txn && rc) { @@ -2167,18 +2199,37 @@ static void do_async_get(void* arg) key.data = UNPACK_BLOB(d->work_buffer, 12); // Allocate a buffer for the output value - value.data = driver_alloc(4096); +#ifdef USE_DRIVER_REALLOC value.ulen = 4096; + value.data = driver_alloc(value.ulen); value.flags = DB_DBT_USERMEM; +#else + value.flags = DB_DBT_MALLOC; +#endif int rc = db->get(db, d->txn, &key, &value, flags); +#ifdef USE_DRIVER_REALLOC while (rc == DB_BUFFER_SMALL) { - // Grow our value buffer and try again - value.data = driver_realloc(value.data, value.size); + // Grow our value buffer value.ulen = value.size; + value.size = 0; + value.data = driver_realloc(value.data, value.ulen); rc = db->get(db, d->txn, &key, &value, flags); } +#endif + // Check CRC - first 4 bytes are CRC of rest of bytes + if (0 == rc) + { + assert(value.size >= 4); + uint32_t calc_crc32 = bdberl_crc32(value.data+4, value.size-4); + uint32_t file_crc32 = *(uint32_t*) value.data; + + if (calc_crc32 != file_crc32) + { + rc = ERROR_INVALID_VALUE; + } + } // Cleanup transaction as necessary if (rc && rc != DB_NOTFOUND && d->txn) @@ -2190,7 +2241,12 @@ static void do_async_get(void* arg) async_cleanup_and_send_kv(d, rc, &key, &value); // Finally, clean up value buffer (driver_send_term made a copy) +#ifdef USE_DRIVER_REALLOC driver_free(value.data); +#else + if (value.data) + free(value.data); +#endif } static void do_async_txnop(void* arg) @@ -2246,6 +2302,19 @@ static void do_async_cursor_get(void* arg) // Execute the operation int rc = d->cursor->get(d->cursor, &key, &value, flags); + // Check CRC - first 4 bytes are CRC of rest of bytes + if (0 == rc) + { + assert(value.size >= 4); + uint32_t calc_crc32 = bdberl_crc32(value.data+4, value.size-4); + uint32_t file_crc32 = *(uint32_t*) value.data; + + if (calc_crc32 != file_crc32) + { + rc = ERROR_INVALID_VALUE; + } + } + // Cleanup as necessary; any sort of failure means we need to close the cursor and abort // the transaction if (rc && rc != DB_NOTFOUND) diff --git a/c_src/bdberl_drv.h b/c_src/bdberl_drv.h index e282b0d..ad5e90a 100644 --- a/c_src/bdberl_drv.h +++ b/c_src/bdberl_drv.h @@ -10,7 +10,7 @@ #include "erl_driver.h" #include "db.h" #include "bdberl_tpool.h" - +#include "bdberl_crc32.h" /** * Driver functions */ @@ -91,6 +91,7 @@ static int bdberl_drv_control(ErlDrvData handle, unsigned int cmd, #define ERROR_DB_ACTIVE (-29007) /* Database is currently active; operation requires otherwise */ #define ERROR_INVALID_CMD (-29008) /* Invalid command code requested */ #define ERROR_INVALID_DB_TYPE (-29009) /* Invalid database type */ +#define ERROR_INVALID_VALUE (-29010) /* Invalid CRC-32 on value */ /** * System information ids diff --git a/src/bdberl.erl b/src/bdberl.erl index b2d7a42..aa37838 100644 --- a/src/bdberl.erl +++ b/src/bdberl.erl @@ -871,7 +871,15 @@ get(Db, Key, Opts) -> case decode_rc(Result) of ok -> receive - {ok, _, Bin} -> {ok, binary_to_term(Bin)}; + {ok, _, Bin} -> + <> = Bin, + case erlang:crc32(Payload) of + Crc -> + {ok, binary_to_term(Payload)}; + CrcOther -> + error_logger:warning_msg("Invalid CRC: ~p ~p\n", [Crc, CrcOther]), + {error, invalid_crc} + end; not_found -> not_found; {error, Reason} -> {error, Reason} end; @@ -2209,9 +2217,14 @@ flag_value(Flag) -> %% do_put(Action, Db, Key, Value, Opts) -> {KeyLen, KeyBin} = to_binary(Key), - {ValLen, ValBin} = to_binary(Value), + {_, ValBin} = to_binary(Value), + {_, ValBin} = to_binary(Value), + Crc = erlang:crc32(ValBin), + FinalValBin = <>, + FinalValBinLen = size(FinalValBin), Flags = process_flags(Opts), - Cmd = <>, + Cmd = <>, <> = erlang:port_control(get_port(), Action, Cmd), case decode_rc(Result) of ok -> @@ -2234,7 +2247,14 @@ do_cursor_move(Direction) -> ok -> receive {ok, KeyBin, ValueBin} -> - {ok, binary_to_term(KeyBin), binary_to_term(ValueBin)}; + <> = ValueBin, + case erlang:crc32(Payload) of + Crc -> + {ok, binary_to_term(KeyBin), binary_to_term(Payload)}; + CrcOther -> + error_logger:warning_msg("Invalid CRC on cursor: ~p ~p\n", [Crc, CrcOther]), + {error, invalid_crc} + end; not_found -> not_found; {error, ReasonCode} ->