fault injection tool for stasis
This commit is contained in:
parent
4bc6d083c6
commit
2533dd921b
6 changed files with 845 additions and 0 deletions
14
test/stasis/fault_injection/CMakeLists.txt
Normal file
14
test/stasis/fault_injection/CMakeLists.txt
Normal file
|
@ -0,0 +1,14 @@
|
|||
ADD_EXECUTABLE(check_faultInjection fitest.c fisubject.c fichecker.c)
|
||||
TARGET_LINK_LIBRARIES(check_faultInjection ${COMMON_LIBRARIES})
|
||||
|
||||
SET_TARGET_PROPERTIES(check_faultInjection PROPERTIES
|
||||
PREFIX ""
|
||||
LINK_FLAGS -Wl,--wrap,malloc,--wrap,fwrite,--wrap,pwrite,--wrap,pwrite64
|
||||
)
|
||||
|
||||
CONFIGURE_FILE(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/config.txt
|
||||
${CMAKE_CURRENT_BINARY_DIR}/config.txt
|
||||
COPYONLY
|
||||
)
|
||||
ADD_TEST(check_faultInjection check_faultInjection)
|
39
test/stasis/fault_injection/config.txt
Normal file
39
test/stasis/fault_injection/config.txt
Normal file
|
@ -0,0 +1,39 @@
|
|||
num_iterations = 5
|
||||
|
||||
num_threads = 1
|
||||
ops_per_thread = 42000
|
||||
ops_per_transaction = 5
|
||||
mallocs_before_failure = 5000
|
||||
log_writes_before_failure = 0 (0)
|
||||
page_writes_before_failure = 0 (0)
|
||||
|
||||
num_threads = 1
|
||||
ops_per_thread = 30000
|
||||
ops_per_transaction = 10
|
||||
mallocs_before_failure = 0
|
||||
log_writes_before_failure = 20000 (35)
|
||||
page_writes_before_failure = 0 (0)
|
||||
|
||||
num_threads = 1
|
||||
ops_per_thread = 32000
|
||||
ops_per_transaction = 20
|
||||
mallocs_before_failure = 0
|
||||
log_writes_before_failure = 0 (0)
|
||||
page_writes_before_failure = 800 (55)
|
||||
|
||||
num_threads = 1
|
||||
ops_per_thread = 33000
|
||||
ops_per_transaction = 13
|
||||
mallocs_before_failure = 10000
|
||||
log_writes_before_failure = 0 (0)
|
||||
page_writes_before_failure = 0 (0)
|
||||
|
||||
num_threads = 1
|
||||
ops_per_thread = 10000
|
||||
ops_per_transaction = 33
|
||||
mallocs_before_failure = 0
|
||||
log_writes_before_failure = 0 (0)
|
||||
page_writes_before_failure = 0 (0)
|
||||
|
||||
|
||||
|
132
test/stasis/fault_injection/fichecker.c
Normal file
132
test/stasis/fault_injection/fichecker.c
Normal file
|
@ -0,0 +1,132 @@
|
|||
/*---
|
||||
This software is copyrighted by the Regents of the University of
|
||||
California and Ashok Sudarsanam. The following terms apply to all files
|
||||
associated with the software unless explicitly disclaimed in
|
||||
individual files.
|
||||
|
||||
The authors hereby grant permission to use, copy, modify, distribute,
|
||||
and license this software and its documentation for any purpose,
|
||||
provided that existing copyright notices are retained in all copies
|
||||
and that this notice is included verbatim in any distributions. No
|
||||
written agreement, license, or royalty fee is required for any of the
|
||||
authorized uses. Modifications to this software may be copyrighted by
|
||||
their authors and need not follow the licensing terms described here,
|
||||
provided that the new terms are clearly indicated on the first page of
|
||||
each file where they apply.
|
||||
|
||||
IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
|
||||
FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
|
||||
ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
|
||||
DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
|
||||
NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
|
||||
THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
|
||||
MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
|
||||
|
||||
GOVERNMENT USE: If you are acquiring this software on behalf of the
|
||||
U.S. government, the Government shall have only "Restricted Rights" in
|
||||
the software and related documentation as defined in the Federal
|
||||
Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
|
||||
acquiring the software on behalf of the Department of Defense, the
|
||||
software shall be classified as "Commercial Computer Software" and the
|
||||
Government shall have only "Restricted Rights" as defined in Clause
|
||||
252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
|
||||
authors grant the U.S. Government and others acting in its behalf
|
||||
permission to use and distribute the software in accordance with the
|
||||
terms specified in this license.
|
||||
---*/
|
||||
|
||||
#include "fitest.h"
|
||||
|
||||
int
|
||||
runChecker() {
|
||||
int hashVal, fd, i, xid, k;
|
||||
int *commitTable = NULL, *insertTable = NULL, tableLength;
|
||||
int *key, **bkey, *value, **bvalue;
|
||||
int keySize, valueSize;
|
||||
lladd_hash_iterator* hashIter;
|
||||
FILE *commitFile, *insertFile;
|
||||
|
||||
printf("\tRunning Checker Process\n"); fflush(stdout);
|
||||
|
||||
readGlobalsFromSharedBuffer();
|
||||
|
||||
/* Read in the hash table recordid from disk. */
|
||||
fd = open("recordid.txt", O_RDONLY, 0777);
|
||||
read(fd, (char*) &hashTable, sizeof(recordid));
|
||||
close(fd);
|
||||
|
||||
/* Open the insert and commit log files. */
|
||||
insertFile = fopen("inserts.txt", "r");
|
||||
commitFile = fopen("commits.txt", "r");
|
||||
|
||||
/* Allocate two tables to keep track of all hash table inserts that were
|
||||
* possibly committed (insert table) and definitely committed (commit table).
|
||||
*/
|
||||
tableLength = baseKey + (numThreads * opsPerThread);
|
||||
insertTable = (int*) calloc(tableLength, sizeof(int));
|
||||
commitTable = (int*) calloc(tableLength, sizeof(int));
|
||||
|
||||
/* Read all the entries from the insert log (commit log) and insert into the
|
||||
* insert table (commit table).
|
||||
*/
|
||||
while (fscanf(insertFile, "%d\n", &hashVal) != EOF) {
|
||||
assert(hashVal < tableLength && insertTable[hashVal] == 0);
|
||||
insertTable[hashVal] = 1;
|
||||
}
|
||||
|
||||
while (fscanf(commitFile, "%d\n", &hashVal) != EOF) {
|
||||
assert(hashVal < tableLength && commitTable[hashVal] == 0);
|
||||
commitTable[hashVal] = 1;
|
||||
}
|
||||
|
||||
/* Iterate over all hash table entries. Report an error if any entry exists in the hash
|
||||
* table that was not committed, i.e.:
|
||||
* (1) entry >= tableLength, or
|
||||
* (2) insertTable[entry] == 0 && commitTable[entry] == 0
|
||||
*/
|
||||
bkey = &key;
|
||||
bvalue = &value;
|
||||
Tinit();
|
||||
xid = Tbegin();
|
||||
|
||||
hashIter = ThashIterator(xid, hashTable, sizeof(int), sizeof(int));
|
||||
while (ThashNext(xid, hashIter, (byte**) bkey, &keySize, (byte**) bvalue, &valueSize)) {
|
||||
k = *key;
|
||||
|
||||
if (k >= tableLength || (insertTable[k] == 0 && commitTable[k] == 0)) {
|
||||
printf("\t\tERROR: <%d> inserted but not committed\n", k); fflush(stdout);
|
||||
return 1;
|
||||
}
|
||||
else {
|
||||
commitTable[k] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
Tabort(xid);
|
||||
Tdeinit();
|
||||
|
||||
/* Now iterate over all commit table entries. If any entry was committed but not
|
||||
* inserted into the hash table (i.e., commitTable[entry] == 1), then report an error.
|
||||
*/
|
||||
for (i = 0; i < tableLength; i++) {
|
||||
if (commitTable[i] == 1) {
|
||||
printf("\t\tERROR: <%d> committed but not inserted\n", i); fflush(stdout);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Close the insert and commit log files. */
|
||||
fclose(insertFile);
|
||||
fclose(commitFile);
|
||||
|
||||
printf("\t\tChecker Process Exiting Normally\n"); fflush(stdout);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
344
test/stasis/fault_injection/fisubject.c
Normal file
344
test/stasis/fault_injection/fisubject.c
Normal file
|
@ -0,0 +1,344 @@
|
|||
/*---
|
||||
This software is copyrighted by the Regents of the University of
|
||||
California and Ashok Sudarsanam. The following terms apply to all files
|
||||
associated with the software unless explicitly disclaimed in
|
||||
individual files.
|
||||
|
||||
The authors hereby grant permission to use, copy, modify, distribute,
|
||||
and license this software and its documentation for any purpose,
|
||||
provided that existing copyright notices are retained in all copies
|
||||
and that this notice is included verbatim in any distributions. No
|
||||
written agreement, license, or royalty fee is required for any of the
|
||||
authorized uses. Modifications to this software may be copyrighted by
|
||||
their authors and need not follow the licensing terms described here,
|
||||
provided that the new terms are clearly indicated on the first page of
|
||||
each file where they apply.
|
||||
|
||||
IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
|
||||
FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
|
||||
ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
|
||||
DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
|
||||
NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
|
||||
THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
|
||||
MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
|
||||
|
||||
GOVERNMENT USE: If you are acquiring this software on behalf of the
|
||||
U.S. government, the Government shall have only "Restricted Rights" in
|
||||
the software and related documentation as defined in the Federal
|
||||
Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
|
||||
acquiring the software on behalf of the Department of Defense, the
|
||||
software shall be classified as "Commercial Computer Software" and the
|
||||
Government shall have only "Restricted Rights" as defined in Clause
|
||||
252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
|
||||
authors grant the U.S. Government and others acting in its behalf
|
||||
permission to use and distribute the software in accordance with the
|
||||
terms specified in this license.
|
||||
---*/
|
||||
|
||||
|
||||
#include "fitest.h"
|
||||
#include <limits.h>
|
||||
|
||||
#define BUFFER_INITIAL_LENGTH 256
|
||||
|
||||
recordid hashTable;
|
||||
int insertLog, commitLog;
|
||||
pthread_mutex_t hashTableMutex;
|
||||
|
||||
int numMallocs = 0;
|
||||
int numLogWrites = 0;
|
||||
int numPageWrites = 0;
|
||||
|
||||
int initHashTable;
|
||||
|
||||
static void* threadFunc(void* arg_ptr);
|
||||
static void writeKeyToBuffer(char** insertBuffer, int* bufferCurrentLength,
|
||||
int* bufferTotalSize, int key);
|
||||
static void writeBufferToLog(int log, char* insertBuffer, int bufferCurrentLength);
|
||||
|
||||
int
|
||||
runSubject() {
|
||||
pthread_t* threads;
|
||||
pthread_attr_t attr;
|
||||
intptr_t k;
|
||||
int xid, fd;
|
||||
|
||||
/* Need the following sleep call for debugging. */
|
||||
//sleep(45);
|
||||
|
||||
printf("\tRunning Subject Process\n"); fflush(stdout);
|
||||
|
||||
readGlobalsFromSharedBuffer();
|
||||
|
||||
initHashTable = 1;
|
||||
|
||||
Tinit();
|
||||
|
||||
if (iteration == 0) {
|
||||
/* Create the transactional hash table data structure. */
|
||||
xid = Tbegin();
|
||||
hashTable = ThashCreate(xid, sizeof(int), sizeof(int));
|
||||
Tcommit(xid);
|
||||
|
||||
/* Write the hash table recordid to a file, so it can be
|
||||
* reloaded during subsequent iterations.
|
||||
*/
|
||||
fd = open("recordid.txt", O_CREAT | O_WRONLY | O_SYNC, 0777);
|
||||
write(fd, (char*) &hashTable, sizeof(recordid));
|
||||
close(fd);
|
||||
}
|
||||
else {
|
||||
/* Otherwise, read in the hash table from disk. */
|
||||
fd = open("recordid.txt", O_RDONLY, 0777);
|
||||
read(fd, (char*) &hashTable, sizeof(recordid));
|
||||
close(fd);
|
||||
}
|
||||
|
||||
initHashTable = 0;
|
||||
|
||||
/* Open the insert and commit log files. The insert-log keeps track of all insertions
|
||||
* that were made to the hash-table, not all of which may have been committed. The
|
||||
* commit-log keeps track of all insertions that were definitely committed.
|
||||
*/
|
||||
insertLog = open("inserts.txt", O_CREAT | O_RDWR | O_APPEND, 0777);
|
||||
commitLog = open("commits.txt", O_CREAT | O_RDWR | O_APPEND, 0777);
|
||||
|
||||
/* Allocate the worker threads. */
|
||||
threads = (pthread_t*) malloc(numThreads * sizeof(pthread_t));
|
||||
|
||||
pthread_attr_init(&attr);
|
||||
pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN);
|
||||
|
||||
pthread_mutex_init(&hashTableMutex, NULL);
|
||||
|
||||
for (k = 0; k < numThreads; k++) {
|
||||
pthread_create(&threads[k], &attr, threadFunc, (void*) k);
|
||||
}
|
||||
|
||||
for (k = 0; k < numThreads; k++) {
|
||||
pthread_join(threads[k], NULL);
|
||||
}
|
||||
|
||||
/* Close the insert and commit log files. */
|
||||
close(insertLog);
|
||||
close(commitLog);
|
||||
|
||||
Tdeinit();
|
||||
|
||||
printf("\t\tSubject Process Exiting Normally\n"); fflush(stdout);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void*
|
||||
threadFunc(void* arg_ptr) {
|
||||
int j, k, startKey, endKey;
|
||||
int xid, count = 0;
|
||||
int bufferCurrentLength, bufferTotalSize;
|
||||
char* insertBuffer;
|
||||
|
||||
/* Allocate the buffer that stores all outstanding hash table insertions. */
|
||||
bufferTotalSize = BUFFER_INITIAL_LENGTH;
|
||||
bufferCurrentLength = 0;
|
||||
insertBuffer = (char*) malloc(bufferTotalSize * sizeof(char));
|
||||
|
||||
xid = Tbegin();
|
||||
|
||||
k = (intptr_t) arg_ptr;
|
||||
|
||||
startKey = baseKey + (k * opsPerThread);
|
||||
endKey = startKey + opsPerThread;
|
||||
|
||||
for (j = startKey; j < endKey; j++) {
|
||||
ThashInsert(xid, hashTable, (byte*)&j, sizeof(int), (byte*)&j, sizeof(int));
|
||||
writeKeyToBuffer(&insertBuffer, &bufferCurrentLength, &bufferTotalSize, j);
|
||||
count++;
|
||||
|
||||
if ((count % opsPerTransaction) == 0) {
|
||||
/* Prior to committing the transaction, write the hash table insertion buffer to
|
||||
* the insert-log so that we can keep track of which insertions were possibly
|
||||
* committed. After the Tcommit() call, write the insertion buffer to the commit-
|
||||
* log so that we can keep track of which insertions were definitely committed.
|
||||
*/
|
||||
writeBufferToLog(insertLog, insertBuffer, bufferCurrentLength);
|
||||
Tcommit(xid);
|
||||
writeBufferToLog(commitLog, insertBuffer, bufferCurrentLength);
|
||||
bufferCurrentLength = 0;
|
||||
|
||||
xid = Tbegin();
|
||||
count = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Prior to committing the transaction, write the hash table insertion buffer to
|
||||
* the insert-log so that we can keep track of which insertions were possibly
|
||||
* committed. After the Tcommit() call, write the insertion buffer to the commit-
|
||||
* log so that we can keep track of which insertions were definitely committed.
|
||||
*/
|
||||
writeBufferToLog(insertLog, insertBuffer, bufferCurrentLength);
|
||||
Tcommit(xid);
|
||||
writeBufferToLog(commitLog, insertBuffer, bufferCurrentLength);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void
|
||||
writeKeyToBuffer(char** insertBuffer, int* bufferCurrentLength,
|
||||
int* bufferTotalSize, int key) {
|
||||
int curLength, totalSize, bytesWritten;
|
||||
|
||||
curLength = *bufferCurrentLength;
|
||||
totalSize = *bufferTotalSize;
|
||||
|
||||
/* Assume we need a maximum of 15 characters for the key, including the terminating
|
||||
* newline and null character. Realloc if there's not enough space in the buffer.
|
||||
*/
|
||||
if ((curLength + 15) > totalSize) {
|
||||
totalSize *= 2;
|
||||
*insertBuffer = (char*) realloc(*insertBuffer, totalSize);
|
||||
*bufferTotalSize = totalSize;
|
||||
}
|
||||
|
||||
bytesWritten = sprintf(*insertBuffer + curLength, "%d\n", key);
|
||||
curLength += bytesWritten;
|
||||
assert(curLength < totalSize);
|
||||
|
||||
*bufferCurrentLength = curLength;
|
||||
}
|
||||
|
||||
static void
|
||||
writeBufferToLog(int log, char* insertBuffer, int bufferCurrentLength) {
|
||||
pthread_mutex_lock(&hashTableMutex);
|
||||
write(log, insertBuffer, bufferCurrentLength);
|
||||
//fsync(log);
|
||||
pthread_mutex_unlock(&hashTableMutex);
|
||||
}
|
||||
|
||||
|
||||
/* The following prototypes are needed to suppress GCC warnings. */
|
||||
void* __real_malloc(int c);
|
||||
size_t __real_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream);
|
||||
ssize_t __real_pwrite(int fildes, const void *buf, size_t nbyte, __off64_t offset);
|
||||
ssize_t __real_pwrite64(int fildes, const void *buf, size_t nbyte, __off64_t offset);
|
||||
|
||||
void*
|
||||
__wrap_malloc(size_t size) {
|
||||
/* fprintf(stderr, "inside wrapper function for malloc\n"); */
|
||||
|
||||
if (!mallocsBeforeFailure || currentMode != SUBJECT_MODE || initHashTable) {
|
||||
return __real_malloc(size);
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&hashTableMutex);
|
||||
numMallocs++;
|
||||
|
||||
if (numMallocs >= mallocsBeforeFailure) {
|
||||
printf("\t\tMALLOC FAILURE: Subject Process Exiting Prematurely\n"); fflush(stdout);
|
||||
|
||||
/* Exit the process. */
|
||||
exit(1);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&hashTableMutex);
|
||||
|
||||
return __real_malloc(size);
|
||||
}
|
||||
|
||||
size_t
|
||||
__wrap_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream) {
|
||||
int updatedSize;
|
||||
|
||||
/* fprintf(stderr, "inside wrapper function for fwrite\n"); */
|
||||
|
||||
if (!logWritesBeforeFailure || currentMode != SUBJECT_MODE || initHashTable) {
|
||||
return __real_fwrite(ptr, size, nitems, stream);
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&hashTableMutex);
|
||||
numLogWrites++;
|
||||
|
||||
if (numLogWrites >= logWritesBeforeFailure) {
|
||||
printf("\t\tFWRITE FAILURE: Subject Process Exiting Prematurely\n"); fflush(stdout);
|
||||
|
||||
/* Possibly write a percentage of the buffer, to simulate the writing of a torn log. */
|
||||
if (logWriteAmountDuringFailure > 0) {
|
||||
updatedSize = (int) ((nitems * logWriteAmountDuringFailure) / 100.0);
|
||||
__real_fwrite(ptr, size, updatedSize, stream);
|
||||
}
|
||||
|
||||
/* Exit the process. */
|
||||
exit(1);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&hashTableMutex);
|
||||
|
||||
return __real_fwrite(ptr, size, nitems, stream);
|
||||
}
|
||||
|
||||
ssize_t
|
||||
__wrap_pwrite(int fildes, const void *buf, size_t nbyte, __off64_t offset) {
|
||||
int updatedSize;
|
||||
|
||||
/* fprintf(stderr, "inside wrapper function for pwrite64\n"); */
|
||||
|
||||
if (!pageWritesBeforeFailure || currentMode != SUBJECT_MODE || initHashTable) {
|
||||
return __real_pwrite(fildes, buf, nbyte, offset);
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&hashTableMutex);
|
||||
numPageWrites++;
|
||||
|
||||
if (numPageWrites >= pageWritesBeforeFailure) {
|
||||
printf("\t\tPWRITE FAILURE: Subject Process Exiting Prematurely\n"); fflush(stdout);
|
||||
|
||||
/* Possibly write a percentage of the buffer, to simulate the writing of a torn page. */
|
||||
if (pageWriteAmountDuringFailure > 0) {
|
||||
updatedSize = (int) ((nbyte * pageWriteAmountDuringFailure) / 100.0);
|
||||
__real_pwrite64(fildes, buf, updatedSize, offset);
|
||||
}
|
||||
|
||||
/* Exit the process. */
|
||||
exit(1);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&hashTableMutex);
|
||||
|
||||
return __real_pwrite(fildes, buf, nbyte, offset);
|
||||
}
|
||||
|
||||
ssize_t
|
||||
__wrap_pwrite64(int fildes, const void *buf, size_t nbyte, __off64_t offset) {
|
||||
int updatedSize;
|
||||
|
||||
/* fprintf(stderr, "inside wrapper function for pwrite64\n"); */
|
||||
|
||||
if (!pageWritesBeforeFailure || currentMode != SUBJECT_MODE || initHashTable) {
|
||||
return __real_pwrite64(fildes, buf, nbyte, offset);
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&hashTableMutex);
|
||||
numPageWrites++;
|
||||
|
||||
if (numPageWrites >= pageWritesBeforeFailure) {
|
||||
printf("\t\tPWRITE64 FAILURE: Subject Process Exiting Prematurely\n"); fflush(stdout);
|
||||
|
||||
/* Possibly write a percentage of the buffer, to simulate the writing of a torn page. */
|
||||
if (pageWriteAmountDuringFailure > 0) {
|
||||
updatedSize = (int) ((nbyte * pageWriteAmountDuringFailure) / 100.0);
|
||||
__real_pwrite64(fildes, buf, updatedSize, offset);
|
||||
}
|
||||
|
||||
/* Exit the process. */
|
||||
exit(1);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&hashTableMutex);
|
||||
|
||||
return __real_pwrite64(fildes, buf, nbyte, offset);
|
||||
}
|
||||
|
228
test/stasis/fault_injection/fitest.c
Normal file
228
test/stasis/fault_injection/fitest.c
Normal file
|
@ -0,0 +1,228 @@
|
|||
/*---
|
||||
This software is copyrighted by the Regents of the University of
|
||||
California and Ashok Sudarsanam. The following terms apply to all files
|
||||
associated with the software unless explicitly disclaimed in
|
||||
individual files.
|
||||
|
||||
The authors hereby grant permission to use, copy, modify, distribute,
|
||||
and license this software and its documentation for any purpose,
|
||||
provided that existing copyright notices are retained in all copies
|
||||
and that this notice is included verbatim in any distributions. No
|
||||
written agreement, license, or royalty fee is required for any of the
|
||||
authorized uses. Modifications to this software may be copyrighted by
|
||||
their authors and need not follow the licensing terms described here,
|
||||
provided that the new terms are clearly indicated on the first page of
|
||||
each file where they apply.
|
||||
|
||||
IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
|
||||
FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
|
||||
ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
|
||||
DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
|
||||
NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
|
||||
THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
|
||||
MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
|
||||
|
||||
GOVERNMENT USE: If you are acquiring this software on behalf of the
|
||||
U.S. government, the Government shall have only "Restricted Rights" in
|
||||
the software and related documentation as defined in the Federal
|
||||
Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
|
||||
acquiring the software on behalf of the Department of Defense, the
|
||||
software shall be classified as "Commercial Computer Software" and the
|
||||
Government shall have only "Restricted Rights" as defined in Clause
|
||||
252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
|
||||
authors grant the U.S. Government and others acting in its behalf
|
||||
permission to use and distribute the software in accordance with the
|
||||
terms specified in this license.
|
||||
---*/
|
||||
|
||||
|
||||
#include "fitest.h"
|
||||
|
||||
testMode currentMode;
|
||||
|
||||
int iteration;
|
||||
int baseKey;
|
||||
int numThreads;
|
||||
int opsPerThread;
|
||||
int opsPerTransaction;
|
||||
int mallocsBeforeFailure;
|
||||
int logWritesBeforeFailure;
|
||||
int pageWritesBeforeFailure;
|
||||
int logWriteAmountDuringFailure;
|
||||
int pageWriteAmountDuringFailure;
|
||||
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
currentMode = INVALID_MODE;
|
||||
|
||||
if (argc == 1) {
|
||||
currentMode = MASTER_MODE;
|
||||
}
|
||||
else {
|
||||
if(!strcmp("-d",argv[1])) {
|
||||
currentMode = INTERACTIVE_MODE;
|
||||
} else {
|
||||
currentMode = (testMode) atoi(argv[1]);
|
||||
}
|
||||
}
|
||||
int err = 0;
|
||||
switch (currentMode) {
|
||||
case MASTER_MODE:
|
||||
err = runMaster(argv[0],0);
|
||||
break;
|
||||
|
||||
case INTERACTIVE_MODE:
|
||||
err = runMaster(argv[0],1);
|
||||
break;
|
||||
|
||||
case SUBJECT_MODE:
|
||||
err = runSubject();
|
||||
break;
|
||||
|
||||
case CHECKER_MODE:
|
||||
err = runChecker();
|
||||
break;
|
||||
|
||||
default:
|
||||
printf("Usage: %s [-d]\n -d: Run interactively (facilitates debugging)\n", argv[0]);
|
||||
err = 1;
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
int
|
||||
runMaster(const char* execname, int interactive) {
|
||||
pid_t pid;
|
||||
int numIters, retval, errorFound = 0;
|
||||
char modeBuf[20];
|
||||
FILE* configFile;
|
||||
|
||||
int fail = 0;
|
||||
|
||||
/* Remove previously generated text files. */
|
||||
unlink("logfile.txt");
|
||||
unlink("storefile.txt");
|
||||
unlink("inserts.txt");
|
||||
unlink("commits.txt");
|
||||
unlink("recordid.txt");
|
||||
unlink("iterinfo.txt");
|
||||
|
||||
configFile = fopen("config.txt", "r");
|
||||
fscanf(configFile, "num_iterations = %d\n\n", &numIters);
|
||||
|
||||
baseKey = 0;
|
||||
|
||||
for (iteration = 0; iteration < numIters; iteration++) {
|
||||
|
||||
printf("*** Testing <Iteration %d> ***\n", iteration); fflush(stdout);
|
||||
|
||||
printf("\tRunning Master Process\n"); fflush(stdout);
|
||||
|
||||
/* Read in the configuration for the current iteration and write to the
|
||||
* file "iter.txt". We will use a shared buffer, instead of a file, to
|
||||
* allow the various processes to communicate with each other.
|
||||
*/
|
||||
fscanf(configFile, "num_threads = %d\n", &numThreads);
|
||||
fscanf(configFile, "ops_per_thread = %d\n", &opsPerThread);
|
||||
fscanf(configFile, "ops_per_transaction = %d\n", &opsPerTransaction);
|
||||
fscanf(configFile, "mallocs_before_failure = %d\n", &mallocsBeforeFailure);
|
||||
fscanf(configFile, "log_writes_before_failure = %d (%d)\n",
|
||||
&logWritesBeforeFailure, &logWriteAmountDuringFailure);
|
||||
fscanf(configFile, "page_writes_before_failure = %d (%d)\n\n",
|
||||
&pageWritesBeforeFailure, &pageWriteAmountDuringFailure);
|
||||
|
||||
writeGlobalsToSharedBuffer();
|
||||
|
||||
/* Fork a new process for the 'Subject'. */
|
||||
pid = fork();
|
||||
|
||||
if (pid == 0) {
|
||||
/* We are in the subject process. */
|
||||
sprintf(modeBuf, "%d", (int) SUBJECT_MODE);
|
||||
if(!interactive) {
|
||||
retval = execl(execname, execname, modeBuf, (char*) 0);
|
||||
perror("Couldn't exec subject process");
|
||||
abort();
|
||||
} else {
|
||||
printf("manually run this, then press enter:\n\t%s %s\n", execname, modeBuf);
|
||||
getc(stdin);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
else {
|
||||
/* We are in the master process, so wait for the subject process to exit. */
|
||||
retval = waitpid(pid, NULL, 0);
|
||||
assert(retval != -1);
|
||||
|
||||
/* Fork a new process for the 'Checker'. */
|
||||
pid = fork();
|
||||
|
||||
if (pid == 0) {
|
||||
/* We are in the checker process. */
|
||||
sprintf(modeBuf, "%d", (int) CHECKER_MODE);
|
||||
if(!interactive) {
|
||||
execl(execname, execname, modeBuf, (char*) 0);
|
||||
perror("Couldn't exec checker process");
|
||||
abort();
|
||||
} else {
|
||||
printf("manually run this, then press enter:\n\t%s %s\n", execname, modeBuf);
|
||||
getc(stdin);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
assert(pid != -1);
|
||||
/* We are in the master process, so wait for the checker process to exit. */
|
||||
retval = waitpid(pid, &errorFound, 0);
|
||||
assert(retval != -1);
|
||||
|
||||
/* Exit the loop if an error was found during the checking process. */
|
||||
if (errorFound) {
|
||||
fail = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
baseKey += (numThreads * opsPerThread);
|
||||
}
|
||||
|
||||
printf("*** Testing Complete: %s ***\n", fail?"FAIL":"PASS"); fflush(stdout);
|
||||
|
||||
fclose(configFile);
|
||||
|
||||
return fail;
|
||||
}
|
||||
|
||||
void
|
||||
writeGlobalsToSharedBuffer() {
|
||||
FILE* iterFile;
|
||||
|
||||
iterFile = fopen("iterinfo.txt", "w");
|
||||
fprintf(iterFile, "%d %d %d %d %d %d %d %d %d %d\n",
|
||||
iteration, baseKey, numThreads, opsPerThread, opsPerTransaction,
|
||||
mallocsBeforeFailure, logWritesBeforeFailure, logWriteAmountDuringFailure,
|
||||
pageWritesBeforeFailure, pageWriteAmountDuringFailure);
|
||||
fclose(iterFile);
|
||||
}
|
||||
|
||||
void
|
||||
readGlobalsFromSharedBuffer() {
|
||||
FILE* iterFile;
|
||||
|
||||
iterFile = fopen("iterinfo.txt", "r");
|
||||
fscanf(iterFile, "%d %d %d %d %d %d %d %d %d %d\n",
|
||||
&iteration, &baseKey, &numThreads, &opsPerThread, &opsPerTransaction,
|
||||
&mallocsBeforeFailure, &logWritesBeforeFailure, &logWriteAmountDuringFailure,
|
||||
&pageWritesBeforeFailure, &pageWriteAmountDuringFailure);
|
||||
fclose(iterFile);
|
||||
}
|
||||
|
||||
|
88
test/stasis/fault_injection/fitest.h
Normal file
88
test/stasis/fault_injection/fitest.h
Normal file
|
@ -0,0 +1,88 @@
|
|||
/*---
|
||||
This software is copyrighted by the Regents of the University of
|
||||
California and Ashok Sudarsanam. The following terms apply to all files
|
||||
associated with the software unless explicitly disclaimed in
|
||||
individual files.
|
||||
|
||||
The authors hereby grant permission to use, copy, modify, distribute,
|
||||
and license this software and its documentation for any purpose,
|
||||
provided that existing copyright notices are retained in all copies
|
||||
and that this notice is included verbatim in any distributions. No
|
||||
written agreement, license, or royalty fee is required for any of the
|
||||
authorized uses. Modifications to this software may be copyrighted by
|
||||
their authors and need not follow the licensing terms described here,
|
||||
provided that the new terms are clearly indicated on the first page of
|
||||
each file where they apply.
|
||||
|
||||
IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
|
||||
FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
|
||||
ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
|
||||
DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
|
||||
NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
|
||||
THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
|
||||
MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
|
||||
|
||||
GOVERNMENT USE: If you are acquiring this software on behalf of the
|
||||
U.S. government, the Government shall have only "Restricted Rights" in
|
||||
the software and related documentation as defined in the Federal
|
||||
Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
|
||||
acquiring the software on behalf of the Department of Defense, the
|
||||
software shall be classified as "Commercial Computer Software" and the
|
||||
Government shall have only "Restricted Rights" as defined in Clause
|
||||
252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
|
||||
authors grant the U.S. Government and others acting in its behalf
|
||||
permission to use and distribute the software in accordance with the
|
||||
terms specified in this license.
|
||||
---*/
|
||||
|
||||
|
||||
#ifndef FI_TEST_H
|
||||
#define FI_TEST_H
|
||||
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <stasis/transactional.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/wait.h>
|
||||
#include <time.h>
|
||||
#include <pthread.h>
|
||||
|
||||
typedef enum {
|
||||
INVALID_MODE = 0,
|
||||
MASTER_MODE,
|
||||
SUBJECT_MODE,
|
||||
CHECKER_MODE,
|
||||
INTERACTIVE_MODE,
|
||||
} testMode;
|
||||
|
||||
int runMaster(const char*, int interactive);
|
||||
int runSubject();
|
||||
int runChecker();
|
||||
void writeGlobalsToSharedBuffer();
|
||||
void readGlobalsFromSharedBuffer();
|
||||
|
||||
extern recordid hashTable;
|
||||
extern testMode currentMode;
|
||||
|
||||
extern int iteration;
|
||||
extern int baseKey;
|
||||
extern int numThreads;
|
||||
extern int opsPerThread;
|
||||
extern int opsPerTransaction;
|
||||
extern int mallocsBeforeFailure;
|
||||
extern int logWritesBeforeFailure;
|
||||
extern int pageWritesBeforeFailure;
|
||||
extern int logWriteAmountDuringFailure;
|
||||
extern int pageWriteAmountDuringFailure;
|
||||
|
||||
#endif
|
Loading…
Reference in a new issue