greg/bdberl
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Incorporated Diz's crc generation/checking patch to bdberl.erl.

Browse source 
Added crc checking before/after BDB accesses in C driver.
Changed get to use malloc DBTs rather than the user supplied buffer
previously.
This commit is contained in:
Jon Meredith 2009-06-04 14:54:27 -06:00 committed by Phillip Toland
parent 700ac5c562
commit dc867e363b
5 changed files with 232 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
c_src/bdberl_crc32.c Normal file
View file

@ -0,0 +1,113 @@
/* -------------------------------------------------------------------
*
* bdberl: CRC-32 checks
* Copyright (c) 2008-9 The Hive. All rights reserved.
*
* ------------------------------------------------------------------- */
#include "bdberl_crc32.h"
/*****************************************************************/
/* */
/* CRC LOOKUP TABLE */
/* ================ */
/* The following CRC lookup table was generated automagically */
/* by the Rocksoft^tm Model CRC Algorithm Table Generation */
/* Program V1.0 using the following model parameters: */
/* */
/* Width : 4 bytes. */
/* Poly : 0x04C11DB7L */
/* Reverse : TRUE. */
/* */
/* For more information on the Rocksoft^tm Model CRC Algorithm, */
/* see the document titled "A Painless Guide to CRC Error */
/* Detection Algorithms" by Ross Williams */
/* (ross@guest.adelaide.edu.au.). This document is likely to be */
/* in the FTP archive "ftp.adelaide.edu.au/pub/rocksoft". */
/* */
/*****************************************************************/
static uint32_t crctable[256] =
{
0x00000000L, 0x77073096L, 0xEE0E612CL, 0x990951BAL,
0x076DC419L, 0x706AF48FL, 0xE963A535L, 0x9E6495A3L,
0x0EDB8832L, 0x79DCB8A4L, 0xE0D5E91EL, 0x97D2D988L,
0x09B64C2BL, 0x7EB17CBDL, 0xE7B82D07L, 0x90BF1D91L,
0x1DB71064L, 0x6AB020F2L, 0xF3B97148L, 0x84BE41DEL,
0x1ADAD47DL, 0x6DDDE4EBL, 0xF4D4B551L, 0x83D385C7L,
0x136C9856L, 0x646BA8C0L, 0xFD62F97AL, 0x8A65C9ECL,
0x14015C4FL, 0x63066CD9L, 0xFA0F3D63L, 0x8D080DF5L,
0x3B6E20C8L, 0x4C69105EL, 0xD56041E4L, 0xA2677172L,
0x3C03E4D1L, 0x4B04D447L, 0xD20D85FDL, 0xA50AB56BL,
0x35B5A8FAL, 0x42B2986CL, 0xDBBBC9D6L, 0xACBCF940L,
0x32D86CE3L, 0x45DF5C75L, 0xDCD60DCFL, 0xABD13D59L,
0x26D930ACL, 0x51DE003AL, 0xC8D75180L, 0xBFD06116L,
0x21B4F4B5L, 0x56B3C423L, 0xCFBA9599L, 0xB8BDA50FL,
0x2802B89EL, 0x5F058808L, 0xC60CD9B2L, 0xB10BE924L,
0x2F6F7C87L, 0x58684C11L, 0xC1611DABL, 0xB6662D3DL,
0x76DC4190L, 0x01DB7106L, 0x98D220BCL, 0xEFD5102AL,
0x71B18589L, 0x06B6B51FL, 0x9FBFE4A5L, 0xE8B8D433L,
0x7807C9A2L, 0x0F00F934L, 0x9609A88EL, 0xE10E9818L,
0x7F6A0DBBL, 0x086D3D2DL, 0x91646C97L, 0xE6635C01L,
0x6B6B51F4L, 0x1C6C6162L, 0x856530D8L, 0xF262004EL,
0x6C0695EDL, 0x1B01A57BL, 0x8208F4C1L, 0xF50FC457L,
0x65B0D9C6L, 0x12B7E950L, 0x8BBEB8EAL, 0xFCB9887CL,
0x62DD1DDFL, 0x15DA2D49L, 0x8CD37CF3L, 0xFBD44C65L,
0x4DB26158L, 0x3AB551CEL, 0xA3BC0074L, 0xD4BB30E2L,
0x4ADFA541L, 0x3DD895D7L, 0xA4D1C46DL, 0xD3D6F4FBL,
0x4369E96AL, 0x346ED9FCL, 0xAD678846L, 0xDA60B8D0L,
0x44042D73L, 0x33031DE5L, 0xAA0A4C5FL, 0xDD0D7CC9L,
0x5005713CL, 0x270241AAL, 0xBE0B1010L, 0xC90C2086L,
0x5768B525L, 0x206F85B3L, 0xB966D409L, 0xCE61E49FL,
0x5EDEF90EL, 0x29D9C998L, 0xB0D09822L, 0xC7D7A8B4L,
0x59B33D17L, 0x2EB40D81L, 0xB7BD5C3BL, 0xC0BA6CADL,
0xEDB88320L, 0x9ABFB3B6L, 0x03B6E20CL, 0x74B1D29AL,
0xEAD54739L, 0x9DD277AFL, 0x04DB2615L, 0x73DC1683L,
0xE3630B12L, 0x94643B84L, 0x0D6D6A3EL, 0x7A6A5AA8L,
0xE40ECF0BL, 0x9309FF9DL, 0x0A00AE27L, 0x7D079EB1L,
0xF00F9344L, 0x8708A3D2L, 0x1E01F268L, 0x6906C2FEL,
0xF762575DL, 0x806567CBL, 0x196C3671L, 0x6E6B06E7L,
0xFED41B76L, 0x89D32BE0L, 0x10DA7A5AL, 0x67DD4ACCL,
0xF9B9DF6FL, 0x8EBEEFF9L, 0x17B7BE43L, 0x60B08ED5L,
0xD6D6A3E8L, 0xA1D1937EL, 0x38D8C2C4L, 0x4FDFF252L,
0xD1BB67F1L, 0xA6BC5767L, 0x3FB506DDL, 0x48B2364BL,
0xD80D2BDAL, 0xAF0A1B4CL, 0x36034AF6L, 0x41047A60L,
0xDF60EFC3L, 0xA867DF55L, 0x316E8EEFL, 0x4669BE79L,
0xCB61B38CL, 0xBC66831AL, 0x256FD2A0L, 0x5268E236L,
0xCC0C7795L, 0xBB0B4703L, 0x220216B9L, 0x5505262FL,
0xC5BA3BBEL, 0xB2BD0B28L, 0x2BB45A92L, 0x5CB36A04L,
0xC2D7FFA7L, 0xB5D0CF31L, 0x2CD99E8BL, 0x5BDEAE1DL,
0x9B64C2B0L, 0xEC63F226L, 0x756AA39CL, 0x026D930AL,
0x9C0906A9L, 0xEB0E363FL, 0x72076785L, 0x05005713L,
0x95BF4A82L, 0xE2B87A14L, 0x7BB12BAEL, 0x0CB61B38L,
0x92D28E9BL, 0xE5D5BE0DL, 0x7CDCEFB7L, 0x0BDBDF21L,
0x86D3D2D4L, 0xF1D4E242L, 0x68DDB3F8L, 0x1FDA836EL,
0x81BE16CDL, 0xF6B9265BL, 0x6FB077E1L, 0x18B74777L,
0x88085AE6L, 0xFF0F6A70L, 0x66063BCAL, 0x11010B5CL,
0x8F659EFFL, 0xF862AE69L, 0x616BFFD3L, 0x166CCF45L,
0xA00AE278L, 0xD70DD2EEL, 0x4E048354L, 0x3903B3C2L,
0xA7672661L, 0xD06016F7L, 0x4969474DL, 0x3E6E77DBL,
0xAED16A4AL, 0xD9D65ADCL, 0x40DF0B66L, 0x37D83BF0L,
0xA9BCAE53L, 0xDEBB9EC5L, 0x47B2CF7FL, 0x30B5FFE9L,
0xBDBDF21CL, 0xCABAC28AL, 0x53B39330L, 0x24B4A3A6L,
0xBAD03605L, 0xCDD70693L, 0x54DE5729L, 0x23D967BFL,
0xB3667A2EL, 0xC4614AB8L, 0x5D681B02L, 0x2A6F2B94L,
0xB40BBE37L, 0xC30C8EA1L, 0x5A05DF1BL, 0x2D02EF8DL
};
/*****************************************************************/
/* End of CRC Lookup Table */
/*****************************************************************/
#define INIT_REFLECTED 0xFFFFFFFF
#define XOROT 0xFFFFFFFF
// Here is the reflected form:
uint32_t bdberl_crc32(const unsigned char *blk_adr, unsigned int blk_len)
{
uint32_t crc = INIT_REFLECTED;
while (blk_len--)
crc = crctable[(crc ^ *blk_adr++) & 0xFFL] ^ (crc >> 8);
return crc ^ XOROT;
}

14
c_src/bdberl_crc32.h Normal file
View file

@ -0,0 +1,14 @@
/* -------------------------------------------------------------------
*
* bdberl: CRC checks
* Copyright (c) 2008-9 The Hive. All rights reserved.
*
* ------------------------------------------------------------------- */
#ifndef _BDBERL_CRC32
#define _BDBERL_CRC32
#include <stdint.h>
uint32_t bdberl_crc32(const unsigned char *blk_adr, unsigned int blk_len);
#endif // _BDBERL_CRC32

89
c_src/bdberl_drv.c
View file

@ -1424,12 +1424,17 @@ static char *rc_to_atom_str(int rc)
switch (rc)
{
// bdberl driver errors
case ERROR_MAX_DBS: return "max_dbs";
case ERROR_ASYNC_PENDING: return "async_pending";
case ERROR_INVALID_DBREF: return "invalid_db";
case ERROR_TXN_OPEN: return "transaction_open";
case ERROR_NO_TXN: return "no_txn";
case ERROR_CURSOR_OPEN: return "cursor_open";
case ERROR_NO_CURSOR: return "no_cursor";
case ERROR_DB_ACTIVE: return "db_active";
case ERROR_INVALID_CMD: return "invalid_cmd";
case ERROR_INVALID_DB_TYPE: return "invalid_db_type";
case ERROR_INVALID_VALUE: return "invalid_value";
// bonafide BDB errors
case DB_BUFFER_SMALL: return "buffer_small";
case DB_DONOTINDEX: return "do_not_index";
@ -1573,10 +1578,24 @@ static void async_cleanup_and_send_kv(PortData* d, int rc, DBT* key, DBT* value)
}
else
{
ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("error"),
ERL_DRV_INT, rc,
ERL_DRV_TUPLE, 2};
driver_send_term(port, pid, response, sizeof(response) / sizeof(response[0]));
// See if this is a standard errno that we have an erlang code for
char *error = rc_to_atom_str(rc);
if (NULL != error)
{
ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("error"),
ERL_DRV_ATOM, driver_mk_atom(error),
ERL_DRV_TUPLE, 2};
driver_send_term(port, pid, response, sizeof(response) / sizeof(response[0]));
}
else
{
ErlDrvTermData response[] = { ERL_DRV_ATOM, driver_mk_atom("error"),
ERL_DRV_ATOM, driver_mk_atom("unknown"),
ERL_DRV_INT, rc,
ERL_DRV_TUPLE, 2,
ERL_DRV_TUPLE, 2};
driver_send_term(port, pid, response, sizeof(response) / sizeof(response[0]));
}
}
}
@ -2121,10 +2140,23 @@ static void do_async_put(void* arg)
value.size = UNPACK_INT(d->work_buffer, 12 + key.size);
value.data = UNPACK_BLOB(d->work_buffer, 12 + key.size + 4);
// Execute the actual put. All databases are opened with AUTO_COMMIT, so if msg->port->txn
// is NULL, the put will still be atomic
int rc = db->put(db, d->txn, &key, &value, flags);
// Check CRC in value payload - first 4 bytes are CRC of rest of bytes
assert(value.size >= 4);
uint32_t calc_crc32 = bdberl_crc32(value.data+4, value.size-4);
uint32_t file_crc32 = *(uint32_t*) value.data;
int rc;
if (calc_crc32 != file_crc32)
{
rc = ERROR_INVALID_VALUE;
}
else
{
// Execute the actual put. All databases are opened with AUTO_COMMIT, so if msg->port->txn
// is NULL, the put will still be atomic
rc = db->put(db, d->txn, &key, &value, flags);
}
// If any error occurs while we have a txn action, abort it
if (d->txn && rc)
{
@ -2167,18 +2199,37 @@ static void do_async_get(void* arg)
key.data = UNPACK_BLOB(d->work_buffer, 12);
// Allocate a buffer for the output value
value.data = driver_alloc(4096);
#ifdef USE_DRIVER_REALLOC
value.ulen = 4096;
value.data = driver_alloc(value.ulen);
value.flags = DB_DBT_USERMEM;
#else
value.flags = DB_DBT_MALLOC;
#endif
int rc = db->get(db, d->txn, &key, &value, flags);
#ifdef USE_DRIVER_REALLOC
while (rc == DB_BUFFER_SMALL)
{
// Grow our value buffer and try again
value.data = driver_realloc(value.data, value.size);
// Grow our value buffer
value.ulen = value.size;
value.size = 0;
value.data = driver_realloc(value.data, value.ulen);
rc = db->get(db, d->txn, &key, &value, flags);
}
#endif
// Check CRC - first 4 bytes are CRC of rest of bytes
if (0 == rc)
{
assert(value.size >= 4);
uint32_t calc_crc32 = bdberl_crc32(value.data+4, value.size-4);
uint32_t file_crc32 = *(uint32_t*) value.data;
if (calc_crc32 != file_crc32)
{
rc = ERROR_INVALID_VALUE;
}
}
// Cleanup transaction as necessary
if (rc && rc != DB_NOTFOUND && d->txn)
@ -2190,7 +2241,12 @@ static void do_async_get(void* arg)
async_cleanup_and_send_kv(d, rc, &key, &value);
// Finally, clean up value buffer (driver_send_term made a copy)
#ifdef USE_DRIVER_REALLOC
driver_free(value.data);
#else
if (value.data)
free(value.data);
#endif
}
static void do_async_txnop(void* arg)
@ -2246,6 +2302,19 @@ static void do_async_cursor_get(void* arg)
// Execute the operation
int rc = d->cursor->get(d->cursor, &key, &value, flags);
// Check CRC - first 4 bytes are CRC of rest of bytes
if (0 == rc)
{
assert(value.size >= 4);
uint32_t calc_crc32 = bdberl_crc32(value.data+4, value.size-4);
uint32_t file_crc32 = *(uint32_t*) value.data;
if (calc_crc32 != file_crc32)
{
rc = ERROR_INVALID_VALUE;
}
}
// Cleanup as necessary; any sort of failure means we need to close the cursor and abort
// the transaction
if (rc && rc != DB_NOTFOUND)

3
c_src/bdberl_drv.h
View file

@ -10,7 +10,7 @@
#include "erl_driver.h"
#include "db.h"
#include "bdberl_tpool.h"
#include "bdberl_crc32.h"
/**
* Driver functions
*/
@ -91,6 +91,7 @@ static int bdberl_drv_control(ErlDrvData handle, unsigned int cmd,
#define ERROR_DB_ACTIVE (-29007) /* Database is currently active; operation requires otherwise */
#define ERROR_INVALID_CMD (-29008) /* Invalid command code requested */
#define ERROR_INVALID_DB_TYPE (-29009) /* Invalid database type */
#define ERROR_INVALID_VALUE (-29010) /* Invalid CRC-32 on value */
/**
* System information ids

28
src/bdberl.erl
View file

@ -871,7 +871,15 @@ get(Db, Key, Opts) ->
case decode_rc(Result) of
ok ->
receive
{ok, _, Bin} -> {ok, binary_to_term(Bin)};
{ok, _, Bin} ->
<<Crc:32/native, Payload/binary>> = Bin,
case erlang:crc32(Payload) of
Crc ->
{ok, binary_to_term(Payload)};
CrcOther ->
error_logger:warning_msg("Invalid CRC: ~p ~p\n", [Crc, CrcOther]),
{error, invalid_crc}
end;
not_found -> not_found;
{error, Reason} -> {error, Reason}
end;
@ -2209,9 +2217,14 @@ flag_value(Flag) ->
%%
do_put(Action, Db, Key, Value, Opts) ->
{KeyLen, KeyBin} = to_binary(Key),
{ValLen, ValBin} = to_binary(Value),
{_, ValBin} = to_binary(Value),
{_, ValBin} = to_binary(Value),
Crc = erlang:crc32(ValBin),
FinalValBin = <<Crc:32/native, ValBin/binary>>,
FinalValBinLen = size(FinalValBin),
Flags = process_flags(Opts),
Cmd = <<Db:32/signed-native, Flags:32/native, KeyLen:32/native, KeyBin/bytes, ValLen:32/native, ValBin/bytes>>,
Cmd = <<Db:32/signed-native, Flags:32/native, KeyLen:32/native, KeyBin/bytes,
FinalValBinLen:32/native, FinalValBin/bytes>>,
<<Result:32/signed-native>> = erlang:port_control(get_port(), Action, Cmd),
case decode_rc(Result) of
ok ->
@ -2234,7 +2247,14 @@ do_cursor_move(Direction) ->
ok ->
receive
{ok, KeyBin, ValueBin} ->
{ok, binary_to_term(KeyBin), binary_to_term(ValueBin)};
<<Crc:32/native, Payload/binary>> = ValueBin,
case erlang:crc32(Payload) of
Crc ->
{ok, binary_to_term(KeyBin), binary_to_term(Payload)};
CrcOther ->
error_logger:warning_msg("Invalid CRC on cursor: ~p ~p\n", [Crc, CrcOther]),
{error, invalid_crc}
end;
not_found ->
not_found;
{error, ReasonCode} ->