mirror of
https://github.com/berkeleydb/je.git
synced 2024-11-15 01:46:24 +00:00
444 lines
16 KiB
Java
444 lines
16 KiB
Java
/*-
|
||
* Copyright (C) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
|
||
*
|
||
* This file was distributed by Oracle as part of a version of Oracle Berkeley
|
||
* DB Java Edition made available at:
|
||
*
|
||
* http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html
|
||
*
|
||
* Please see the LICENSE file included in the top-level directory of the
|
||
* appropriate version of Oracle Berkeley DB Java Edition for a copy of the
|
||
* license and additional information.
|
||
*/
|
||
|
||
import java.io.File;
|
||
import java.math.BigInteger;
|
||
import java.security.SecureRandom;
|
||
import java.util.Arrays;
|
||
|
||
import com.sleepycat.je.Cursor;
|
||
import com.sleepycat.je.Database;
|
||
import com.sleepycat.je.DatabaseConfig;
|
||
import com.sleepycat.je.DatabaseEntry;
|
||
import com.sleepycat.je.DatabaseException;
|
||
import com.sleepycat.je.LockConflictException;
|
||
import com.sleepycat.je.Environment;
|
||
import com.sleepycat.je.EnvironmentConfig;
|
||
import com.sleepycat.je.LockMode;
|
||
import com.sleepycat.je.OperationStatus;
|
||
|
||
/**
|
||
* A large database with random key distribution has lots of IN waste,
|
||
* especially if records are small; this creates a worst-case scenario for the
|
||
* cleaner and also possibly for the evictor. Simulate such an application and
|
||
* measure how well the cleaner and evictor keep up.
|
||
*
|
||
* Some commonly used command lines for running this program are:
|
||
*
|
||
* # Init new DB, causes duplicates to be created and deleted [#15588]
|
||
* java BigDW -h HOME -init -dupdel
|
||
*
|
||
* Each transaction does the following in "grow" mode. In "no grow" mode, it
|
||
* does one less insert, keeping the total number of keys constant.
|
||
*
|
||
* 2 inserts, 1 delete, 1 update, 10 reads
|
||
*
|
||
* The delete and update operations include a read to find the record.
|
||
*
|
||
*/
|
||
public class BigDW implements Runnable {
|
||
|
||
private String homeDir = "tmp";
|
||
private Environment env;
|
||
private Database refDB;
|
||
private Database testDB;
|
||
private boolean done;
|
||
private int nDeadlocks;
|
||
private boolean init;
|
||
private boolean verbose;
|
||
private boolean dupDel;
|
||
private int nTransactions;
|
||
private int nMaxTransactions = 20000;
|
||
private int nThreads = 4;
|
||
|
||
private int subDir = 0;
|
||
private int keySize = 10;
|
||
private int dataSize = 10;
|
||
private int nReadsPerWrite = 1;
|
||
private int maxRetries = 100;
|
||
private float totalSecs;
|
||
private float throughput;
|
||
private SecureRandom random = new SecureRandom();
|
||
private long startTime;
|
||
private long time;
|
||
private long mainCacheSize = 20000000;
|
||
|
||
public static void main(String args[]) {
|
||
try {
|
||
new BigDW().run(args);
|
||
System.exit(0);
|
||
} catch (Throwable e) {
|
||
e.printStackTrace(System.out);
|
||
System.exit(1);
|
||
}
|
||
}
|
||
|
||
/* Output command-line input arguments to log. */
|
||
private void printArgs(String[] args) {
|
||
System.out.print("\nCommand line arguments:");
|
||
for (String arg : args) {
|
||
System.out.print(' ');
|
||
System.out.print(arg);
|
||
}
|
||
System.out.println();
|
||
}
|
||
|
||
private void usage(String error) {
|
||
|
||
if (error != null) {
|
||
System.err.println(error);
|
||
}
|
||
System.err.println
|
||
("java " + getClass().getName() + '\n' +
|
||
" [-h <homeDir>] [-v] [-init] [-dupdel]\n" +
|
||
" [-txns <maxTxns>]\n");
|
||
System.exit(1);
|
||
}
|
||
|
||
private void run(String args[]) throws Exception {
|
||
|
||
try {
|
||
if (args.length == 0) {
|
||
throw new IllegalArgumentException();
|
||
}
|
||
/* Parse command-line input arguments. */
|
||
for (int i = 0; i < args.length; i += 1) {
|
||
String arg = args[i];
|
||
boolean moreArgs = i < args.length - 1;
|
||
if (arg.equals("-v")) {
|
||
verbose = true;
|
||
} else if (arg.equals("-dupdel")) {
|
||
dupDel = true;
|
||
} else if (arg.equals("-h") && moreArgs) {
|
||
homeDir = args[++i];
|
||
} else if (arg.equals("-init")) {
|
||
init = true;
|
||
} else if (arg.equals("-txns") && moreArgs) {
|
||
nMaxTransactions = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-threads") && moreArgs) {
|
||
nThreads = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-subDir") && moreArgs) {
|
||
subDir = Integer.parseInt(args[++i]);
|
||
} else {
|
||
usage("Unknown arg: " + arg);
|
||
}
|
||
}
|
||
printArgs(args);
|
||
} catch (IllegalArgumentException e) {
|
||
usage("IllegalArguments! ");
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
|
||
openEnv();
|
||
startTime = System.currentTimeMillis();
|
||
|
||
Thread[] threads = new Thread[nThreads];
|
||
for (int i = 0; i < nThreads; i += 1) {
|
||
threads[i] = new Thread(this);
|
||
threads[i].start();
|
||
Thread.sleep(1000); /* Stagger threads. */
|
||
}
|
||
for (int i = 0; i < nThreads; i += 1) {
|
||
if (threads[i] != null) {
|
||
threads[i].join();
|
||
}
|
||
}
|
||
|
||
time = System.currentTimeMillis();
|
||
closeEnv();
|
||
|
||
totalSecs = (float) (time - startTime) / 1000;
|
||
throughput = (float) nTransactions / totalSecs;
|
||
if (verbose) {
|
||
System.out.println("\nTotal seconds: " + totalSecs +
|
||
" txn/sec: " + throughput);
|
||
}
|
||
}
|
||
|
||
private void openEnv() throws Exception {
|
||
EnvironmentConfig envConfig = new EnvironmentConfig();
|
||
envConfig.setTransactional(true);
|
||
envConfig.setAllowCreate(init);
|
||
envConfig.setCacheSize(mainCacheSize);
|
||
if (subDir > 0) {
|
||
envConfig.setConfigParam
|
||
(EnvironmentConfig.LOG_N_DATA_DIRECTORIES, subDir + "");
|
||
Utils.createSubDirs(new File(homeDir), subDir);
|
||
}
|
||
env = new Environment(new File(homeDir), envConfig);
|
||
|
||
DatabaseConfig dbConfig = new DatabaseConfig();
|
||
dbConfig.setAllowCreate(init);
|
||
dbConfig.setExclusiveCreate(init);
|
||
dbConfig.setSortedDuplicates(dupDel);
|
||
refDB = env.openDatabase(null, "BigDWRef", dbConfig);
|
||
|
||
dbConfig.setDeferredWrite(true);
|
||
testDB = env.openDatabase(null, "BigDWTest", dbConfig);
|
||
|
||
compare();
|
||
}
|
||
|
||
private void closeEnv()
|
||
throws Exception {
|
||
|
||
refDB.close();
|
||
testDB.sync();
|
||
testDB.close();
|
||
env.close();
|
||
}
|
||
|
||
public void run() {
|
||
|
||
DatabaseEntry data = new DatabaseEntry();
|
||
DatabaseEntry key = new DatabaseEntry();
|
||
byte[] lastInsertKey = null;
|
||
|
||
while (!done) {
|
||
|
||
/* JE-only begin */
|
||
try {
|
||
|
||
/* Perform the transaction. */
|
||
for (int retry = 0;; retry += 1) {
|
||
Cursor refCursor = refDB.openCursor(null, null);
|
||
Cursor testCursor = testDB.openCursor(null, null);
|
||
|
||
try {
|
||
if (init) {
|
||
key.setData(lastInsertKey);
|
||
insert(refCursor, testCursor, key, data);
|
||
lastInsertKey = copyData(key);
|
||
}
|
||
|
||
/* Insert */
|
||
key.setData(lastInsertKey);
|
||
insert(refCursor, testCursor, key, data);
|
||
lastInsertKey = copyData(key);
|
||
|
||
/* Dup-key insert. */
|
||
byte[] dupDataBA = copyData(data);
|
||
for (int i = 0; i < 5; i++) {
|
||
dupDataBA[0]++;
|
||
DatabaseEntry dupData =
|
||
new DatabaseEntry(dupDataBA);
|
||
OperationStatus status1 =
|
||
refCursor.put(key, dupData);
|
||
@SuppressWarnings("unused")
|
||
boolean insertDone1 = checkInsertStatus(status1);
|
||
if (status1 != OperationStatus.SUCCESS) {
|
||
throw new RuntimeException("insert1 " +
|
||
status1);
|
||
}
|
||
OperationStatus status2 =
|
||
testCursor.put(key, dupData);
|
||
if (status2 != OperationStatus.SUCCESS) {
|
||
throw new RuntimeException("insert2 " +
|
||
status2);
|
||
}
|
||
@SuppressWarnings("unused")
|
||
boolean insertDone2 = checkInsertStatus(status2);
|
||
}
|
||
|
||
/* Delete */
|
||
getRandom(refCursor, "BigDWRef",
|
||
testCursor, "BigDWTest",
|
||
key, data, LockMode.RMW);
|
||
DatabaseEntry dummy1 = new DatabaseEntry();
|
||
DatabaseEntry dummy2 = new DatabaseEntry();
|
||
while (refCursor.delete() ==
|
||
OperationStatus.SUCCESS &&
|
||
refCursor.getNextDup
|
||
(dummy1, dummy2, null) ==
|
||
OperationStatus.SUCCESS) {
|
||
}
|
||
while (testCursor.delete() ==
|
||
OperationStatus.SUCCESS &&
|
||
refCursor.getNextDup
|
||
(dummy1, dummy2, null) ==
|
||
OperationStatus.SUCCESS) {
|
||
}
|
||
|
||
/* Read */
|
||
for (int i = 0; i < nReadsPerWrite; i += 1) {
|
||
getRandom(refCursor, "BigDWRef",
|
||
testCursor, "BigDWTest",
|
||
key, data, LockMode.RMW);
|
||
}
|
||
refCursor.close();
|
||
testCursor.close();
|
||
nTransactions += 1;
|
||
if (nMaxTransactions != 0 &&
|
||
nTransactions >= nMaxTransactions) {
|
||
done = true;
|
||
}
|
||
break;
|
||
} catch (LockConflictException e) {
|
||
refCursor.close();
|
||
testCursor.close();
|
||
if (retry >= maxRetries) {
|
||
throw e;
|
||
}
|
||
/* Break deadlock cycle with a small sleep. */
|
||
Thread.sleep(5);
|
||
nDeadlocks += 1;
|
||
}
|
||
} /* for */
|
||
|
||
} catch (Throwable e) {
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
}
|
||
}
|
||
|
||
private void checkStatus(OperationStatus status)
|
||
throws Exception {
|
||
if (status != OperationStatus.SUCCESS) {
|
||
throw new Exception("problemStatus = " + status);
|
||
}
|
||
}
|
||
|
||
private void compare()
|
||
throws Exception {
|
||
|
||
DatabaseEntry refKey = new DatabaseEntry();
|
||
DatabaseEntry refData = new DatabaseEntry();
|
||
DatabaseEntry testKey = new DatabaseEntry();
|
||
DatabaseEntry testData = new DatabaseEntry();
|
||
|
||
Cursor refCursor = refDB.openCursor(null, null);
|
||
Cursor testCursor = testDB.openCursor(null, null);
|
||
|
||
System.out.println("Compare starts");
|
||
try {
|
||
while (refCursor.getNext(refKey, refData, LockMode.DEFAULT) ==
|
||
OperationStatus.SUCCESS) {
|
||
checkStatus(testCursor.getNext(testKey, testData,
|
||
LockMode.DEFAULT));
|
||
|
||
if (!Arrays.equals(refKey.getData(),
|
||
testKey.getData())) {
|
||
throw new Exception("Keys don't match");
|
||
}
|
||
|
||
if (!Arrays.equals(refData.getData(),
|
||
testData.getData())) {
|
||
throw new Exception("Data don't match");
|
||
}
|
||
}
|
||
|
||
if (testCursor.getNext(testKey, testData, LockMode.DEFAULT) !=
|
||
OperationStatus.NOTFOUND) {
|
||
throw new Exception("testCursor has extra data");
|
||
}
|
||
} finally {
|
||
refCursor.close();
|
||
testCursor.close();
|
||
}
|
||
System.out.println("Compare ends");
|
||
}
|
||
|
||
private void insert(Cursor c1, Cursor c2,
|
||
DatabaseEntry key, DatabaseEntry data)
|
||
throws DatabaseException {
|
||
|
||
makeData(data);
|
||
boolean insertDone1 = false;
|
||
while (!insertDone1) {
|
||
makeInsertKey(key);
|
||
OperationStatus status1 = c1.putNoOverwrite(key, data);
|
||
insertDone1 = checkInsertStatus(status1);
|
||
OperationStatus status2 = c2.putNoOverwrite(key, data);
|
||
boolean insertDone2 = checkInsertStatus(status2);
|
||
assert insertDone1 == insertDone2 :
|
||
"status1=" + status1 +
|
||
" status2=" + status2;
|
||
}
|
||
}
|
||
|
||
private boolean checkInsertStatus(OperationStatus status) {
|
||
if (status == OperationStatus.KEYEXIST) {
|
||
System.out.println("****** Duplicate random key.");
|
||
return false; // try again.
|
||
} else {
|
||
if (status != OperationStatus.SUCCESS) {
|
||
System.out.println
|
||
("Unexpected return value from insert(): " + status);
|
||
}
|
||
return true; // end one way or another
|
||
}
|
||
}
|
||
|
||
private void getRandom(Cursor c1, String db1,
|
||
Cursor c2, String db2,
|
||
DatabaseEntry key, DatabaseEntry data,
|
||
LockMode lockMode)
|
||
throws DatabaseException {
|
||
|
||
makeRandomKey(key);
|
||
getRandomWork(c1, db1, key, data, lockMode);
|
||
getRandomWork(c2, db2, key, data, lockMode);
|
||
}
|
||
|
||
private void getRandomWork(Cursor c,
|
||
String dbName,
|
||
DatabaseEntry key,
|
||
DatabaseEntry data,
|
||
LockMode lockMode)
|
||
throws DatabaseException {
|
||
|
||
OperationStatus status = c.getSearchKeyRange(key, data, lockMode);
|
||
if (status == OperationStatus.NOTFOUND) {
|
||
status = c.getLast(key, data, lockMode);
|
||
if (status != OperationStatus.SUCCESS) {
|
||
System.out.println
|
||
("Unexpected return value from " + dbName +
|
||
".getRandomWork(): " + status);
|
||
}
|
||
}
|
||
}
|
||
|
||
private void makeInsertKey(DatabaseEntry key) {
|
||
if (key.getData() != null) {
|
||
BigInteger num = new BigInteger(copyData(key));
|
||
num = num.add(BigInteger.ONE);
|
||
key.setData(num.toByteArray());
|
||
} else {
|
||
makeRandomKey(key);
|
||
}
|
||
}
|
||
|
||
private void makeRandomKey(DatabaseEntry key) {
|
||
byte[] bytes = new byte[keySize];
|
||
random.nextBytes(bytes);
|
||
key.setData(bytes);
|
||
}
|
||
|
||
private void makeData(DatabaseEntry data) {
|
||
|
||
byte[] bytes = new byte[dataSize];
|
||
for (int i = 0; i < bytes.length; i += 1) {
|
||
bytes[i] = (byte) i;
|
||
}
|
||
data.setData(bytes);
|
||
}
|
||
|
||
private byte[] copyData(DatabaseEntry data) {
|
||
|
||
byte[] buf = new byte[data.getSize()];
|
||
System.arraycopy(data.getData(), data.getOffset(), buf, 0, buf.length);
|
||
return buf;
|
||
}
|
||
}
|