mirror of
https://github.com/berkeleydb/je.git
synced 2024-07-06 09:57:18 +00:00
625 lines
23 KiB
Java
625 lines
23 KiB
Java
/*-
|
||
* Copyright (C) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
|
||
*
|
||
* This file was distributed by Oracle as part of a version of Oracle Berkeley
|
||
* DB Java Edition made available at:
|
||
*
|
||
* http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html
|
||
*
|
||
* Please see the LICENSE file included in the top-level directory of the
|
||
* appropriate version of Oracle Berkeley DB Java Edition for a copy of the
|
||
* license and additional information.
|
||
*/
|
||
|
||
import java.io.File;
|
||
import java.math.BigInteger;
|
||
import java.security.SecureRandom;
|
||
|
||
import com.sleepycat.bind.tuple.TupleInput;
|
||
import com.sleepycat.bind.tuple.TupleOutput;
|
||
import com.sleepycat.je.CacheMode;
|
||
import com.sleepycat.je.Cursor;
|
||
import com.sleepycat.je.Database;
|
||
import com.sleepycat.je.DatabaseConfig;
|
||
import com.sleepycat.je.DatabaseEntry;
|
||
import com.sleepycat.je.DatabaseException;
|
||
import com.sleepycat.je.DbInternal;
|
||
import com.sleepycat.je.Environment;
|
||
import com.sleepycat.je.EnvironmentConfig;
|
||
import com.sleepycat.je.EnvironmentStats;
|
||
import com.sleepycat.je.LockConflictException;
|
||
import com.sleepycat.je.LockMode;
|
||
import com.sleepycat.je.OperationStatus;
|
||
import com.sleepycat.je.StatsConfig;
|
||
import com.sleepycat.je.Transaction;
|
||
import com.sleepycat.je.dbi.MemoryBudget;
|
||
|
||
/**
|
||
* A large database with random key distribution has lots of IN waste,
|
||
* especially if records are small; this creates a worst-case scenario for the
|
||
* cleaner and also possibly for the evictor. Simulate such an application and
|
||
* measure how well the cleaner and evictor keep up.
|
||
*
|
||
* Some commonly used command lines for running this program are:
|
||
*
|
||
* # Init new DB, then do updates forever.
|
||
* java BigRandom -h HOME -init
|
||
*
|
||
* # Do updates on an existing DB forever.
|
||
* java BigRandom -h HOME
|
||
*
|
||
* # Init new DB, then stop and print total rate (MOST COMMON OPTION)
|
||
* java BigRandom -h HOME -initonly
|
||
*
|
||
* # -locality N adds locality of reference for N transactions.
|
||
* java BigRandom -h HOME -initonly -locality 5
|
||
*
|
||
* # -nosync speeds things up quite a bit
|
||
* java BigRandom -h HOME -initonly -locality 5 -nosync
|
||
*
|
||
* Each transaction does the following in "grow" mode. In "no grow" mode, it
|
||
* does one less insert, keeping the total number of keys constant.
|
||
*
|
||
* 2 inserts, 1 delete, 1 update, 10 reads
|
||
*
|
||
* The delete and update operations include a read to find the record.
|
||
*
|
||
* Every operation uses a random key, unless the -locality option is used. If
|
||
* "-locality 100" is specified, each thread will perform 100 transactions by
|
||
* incrementing the insertion key rather than generating a random number. Then
|
||
* a random number is generated as the next starting key. This is done per
|
||
* thread, so each thread will be working in a different key area.
|
||
*/
|
||
public class BigRandom implements Runnable {
|
||
|
||
private String homeDir = "tmp";
|
||
private Environment env;
|
||
private Database db;
|
||
private boolean done;
|
||
private int nDeadlocks;
|
||
private boolean init;
|
||
private boolean initOnly;
|
||
private boolean fastInit;
|
||
private boolean verbose;
|
||
private boolean sequentialKeys;
|
||
private boolean noSync;
|
||
private int nMaxKeys = 10000000;
|
||
private long nKeys;
|
||
private long sequence;
|
||
private int nTransactions;
|
||
private int nMaxTransactions;
|
||
private int nThreads = 4;
|
||
private int oneThreadKeys;
|
||
private long traceInterval = 10000; // 10 seconds
|
||
private boolean preload;
|
||
private int maxLocalKeyTxns;
|
||
private int keySize = 10;
|
||
private int dataSize = 20;
|
||
private int nReadsPerWrite = 10;
|
||
private int maxRetries = 100;
|
||
private SecureRandom random = new SecureRandom();
|
||
private long startTime;
|
||
private long priorTime = startTime;
|
||
private int priorTxns;
|
||
private int[] tpTxns = new int[120]; // 120 * 10 sec = ~20 minutes worth
|
||
private long[] tpMillis = new long[tpTxns.length];
|
||
private int tpIndex = tpTxns.length - 1;
|
||
private int tpMaxIndex;
|
||
private long tpTotalTxns;
|
||
private long tpTotalMillis;
|
||
private int thisTxns;
|
||
private int thisSecs;
|
||
private int thisTp;
|
||
private int avgTp;
|
||
private long time;
|
||
private int totalSecs;
|
||
private int subDir = 0;
|
||
|
||
public static void main(String args[]) {
|
||
try {
|
||
new BigRandom().run(args);
|
||
System.exit(0);
|
||
} catch (Throwable e) {
|
||
e.printStackTrace(System.out);
|
||
System.exit(1);
|
||
}
|
||
}
|
||
|
||
private void run(String args[])
|
||
throws Exception {
|
||
|
||
for (int i = 0; i < args.length; i += 1) {
|
||
String arg = args[i];
|
||
boolean moreArgs = i < args.length - 1;
|
||
if (arg.equals("-v")) {
|
||
verbose = true;
|
||
} else if (arg.equals("-seq")) {
|
||
sequentialKeys = true;
|
||
} else if (arg.equals("-nosync")) {
|
||
noSync = true;
|
||
} else if (arg.equals("-h") && moreArgs) {
|
||
homeDir = args[++i];
|
||
} else if (arg.equals("-preload")) {
|
||
preload = true;
|
||
} else if (arg.equals("-init")) {
|
||
init = true;
|
||
} else if (arg.equals("-initonly")) {
|
||
init = true;
|
||
initOnly = true;
|
||
} else if (arg.equals("-fastinit")) {
|
||
init = true;
|
||
fastInit = true;
|
||
initOnly = true;
|
||
} else if (arg.equals("-keys") && moreArgs) {
|
||
nMaxKeys = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-txns") && moreArgs) {
|
||
nMaxTransactions = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-threads") && moreArgs) {
|
||
nThreads = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-onethreadkeys") && moreArgs) {
|
||
oneThreadKeys = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-locality") && moreArgs) {
|
||
maxLocalKeyTxns = Integer.parseInt(args[++i]);
|
||
} else if (arg.equals("-subDir") && moreArgs) {
|
||
subDir = Integer.parseInt(args[++i]);
|
||
} else {
|
||
usage("Unknown arg: " + arg);
|
||
}
|
||
}
|
||
openEnv();
|
||
printArgs(args);
|
||
printLegend();
|
||
if (sequentialKeys) {
|
||
sequence = getLastSequence();
|
||
}
|
||
if (preload) {
|
||
doPreload();
|
||
}
|
||
StatsConfig statsConfig = new StatsConfig();
|
||
statsConfig.setFast(true);
|
||
statsConfig.setClear(true);
|
||
startTime = System.currentTimeMillis();
|
||
priorTime = startTime;
|
||
|
||
Thread[] threads = new Thread[nThreads];
|
||
if (oneThreadKeys > 0) {
|
||
threads[0] = new Thread(this);
|
||
threads[0].start();
|
||
} else {
|
||
for (int i = 0; i < nThreads; i += 1) {
|
||
threads[i] = new Thread(this);
|
||
threads[i].start();
|
||
Thread.sleep(1000); /* Stagger threads. */
|
||
}
|
||
}
|
||
|
||
while (!done) {
|
||
Thread.sleep(traceInterval);
|
||
calcThroughput();
|
||
/* JE-only begin */
|
||
EnvironmentStats stats = env.getStats(statsConfig);
|
||
MemoryBudget mb =
|
||
DbInternal.getNonNullEnvImpl(env).getMemoryBudget();
|
||
int inListSize =
|
||
DbInternal.getNonNullEnvImpl(env).getInMemoryINs().
|
||
getSize();
|
||
System.out.println("\nsec: " + totalSecs + ',' + thisSecs +
|
||
" txn: " + thisTxns + ',' +
|
||
thisTp + ',' + avgTp +
|
||
" keys: " + nKeys +
|
||
" dlck: " + nDeadlocks +
|
||
" buf: " +
|
||
stats.getNNotResident() + ',' +
|
||
stats.getNCacheMiss() +
|
||
"\ncleaner: " +
|
||
stats.getNCleanerEntriesRead() + ',' +
|
||
stats.getNCleanerRuns() + ',' +
|
||
stats.getNCleanerDeletions() + ',' +
|
||
stats.getCleanerBacklog() +
|
||
" evict: " +
|
||
stats.getNBINsStripped() + ',' +
|
||
stats.getNNodesExplicitlyEvicted() + ',' +
|
||
mb.getCacheMemoryUsage() + ',' +
|
||
inListSize +
|
||
" ckpt: " +
|
||
stats.getNCheckpoints() + ',' +
|
||
stats.getNFullINFlush() + ',' +
|
||
stats.getNFullBINFlush() + ',' +
|
||
stats.getNDeltaINFlush());
|
||
/* JE-only end */
|
||
nDeadlocks = 0;
|
||
|
||
if (oneThreadKeys > 0 && oneThreadKeys >= nKeys) {
|
||
for (int i = 1; i < nThreads; i += 1) {
|
||
threads[i] = new Thread(this);
|
||
threads[i].start();
|
||
Thread.sleep(1000); /* Stagger threads. */
|
||
}
|
||
oneThreadKeys = 0;
|
||
}
|
||
}
|
||
|
||
for (int i = 0; i < nThreads; i += 1) {
|
||
if (threads[i] != null) {
|
||
threads[i].join();
|
||
}
|
||
}
|
||
|
||
time = System.currentTimeMillis();
|
||
totalSecs = (int) ((time - startTime) / 1000);
|
||
System.out.println("\nTotal seconds: " + totalSecs +
|
||
" txn/sec: " + (nTransactions / totalSecs));
|
||
closeEnv();
|
||
}
|
||
|
||
private void calcThroughput() {
|
||
|
||
time = System.currentTimeMillis();
|
||
totalSecs = (int) ((time - startTime) / 1000);
|
||
int txns = nTransactions;
|
||
thisTxns = txns - priorTxns;
|
||
int thisMillis = (int) (time - priorTime);
|
||
thisSecs = thisMillis / 1000;
|
||
thisTp = thisTxns / thisSecs;
|
||
|
||
tpIndex += 1;
|
||
if (tpIndex == tpTxns.length) {
|
||
tpIndex = 0;
|
||
}
|
||
tpTotalTxns += thisTxns;
|
||
tpTotalTxns -= tpTxns[tpIndex];
|
||
tpTotalMillis += thisMillis;
|
||
tpTotalMillis -= tpMillis[tpIndex];
|
||
tpTxns[tpIndex] = thisTxns;
|
||
tpMillis[tpIndex] = thisMillis;
|
||
if (tpMaxIndex < tpTxns.length) {
|
||
tpMaxIndex = tpIndex + 1;
|
||
}
|
||
avgTp = (int) ((tpTotalTxns / (tpTotalMillis / 1000)));
|
||
|
||
priorTxns = txns;
|
||
priorTime = time;
|
||
}
|
||
|
||
private void printArgs(String[] args)
|
||
throws DatabaseException {
|
||
|
||
System.out.print("Command line arguments:");
|
||
for (String arg : args) {
|
||
System.out.print(' ');
|
||
System.out.print(arg);
|
||
}
|
||
System.out.println();
|
||
System.out.println();
|
||
System.out.println("Environment configuration:");
|
||
System.out.println(env.getConfig());
|
||
System.out.println();
|
||
}
|
||
|
||
private void printLegend() {
|
||
|
||
/* JE-only begin */
|
||
System.out.println(
|
||
"Legend:\n" +
|
||
"sec: <totalSeconds>,<runSeconds>\n" +
|
||
"txn: <txns>,<txnPerSec>,<runningAvgTxnPerSec>\n" +
|
||
"keys: <totalKeys>\n" +
|
||
"dlck: <deadlocks>\n" +
|
||
"buf: <notResident>,<cacheMisses>\n" +
|
||
"clean: <entriesRead>,<filesCleaned>,<filesDeleted>,<backlog>\n" +
|
||
"evict: <binsStripped>,<nodesEvicted>,<cacheSize>,<INListSize>\n" +
|
||
"ckpt: <checkpointsStarted>,<fullINs>,<fullBINs>,<deltaINs>");
|
||
/* JE-only end */
|
||
}
|
||
|
||
private void usage(String error) {
|
||
|
||
if (error != null) {
|
||
System.err.println(error);
|
||
}
|
||
System.err.println
|
||
("java " + getClass().getName() + '\n' +
|
||
" [-h <homeDir>] [-v] [-init | -initonly | -fastinit]\n" +
|
||
" [-keys <maxKeys>] [-txns <maxTxns>] [-seq]\n" +
|
||
" [-threads <appThreads>] [-onethreadkeys <nKeys>]\n" +
|
||
" [-locality <nTxns>] [-nosync] [-preload]");
|
||
System.exit(2);
|
||
}
|
||
|
||
private void openEnv() throws Exception {
|
||
|
||
EnvironmentConfig envConfig = new EnvironmentConfig();
|
||
envConfig.setTransactional(true);
|
||
envConfig.setAllowCreate(init);
|
||
envConfig.setCacheMode(CacheMode.EVICT_LN);
|
||
envConfig.setConfigParam(EnvironmentConfig.MAX_OFF_HEAP_MEMORY,
|
||
"" + (50 * 1024 * 1024));
|
||
|
||
if (noSync) {
|
||
envConfig.setTxnNoSync(true);
|
||
}
|
||
|
||
if (subDir > 0) {
|
||
envConfig.setConfigParam
|
||
(EnvironmentConfig.LOG_N_DATA_DIRECTORIES, subDir + "");
|
||
Utils.createSubDirs(new File(homeDir), subDir);
|
||
}
|
||
|
||
long startTime = System.currentTimeMillis();
|
||
env = new Environment(new File(homeDir), envConfig);
|
||
long endTime = System.currentTimeMillis();
|
||
System.out.println("Recovery time: " + ((endTime - startTime) / 1000));
|
||
System.out.println();
|
||
|
||
DatabaseConfig dbConfig = new DatabaseConfig();
|
||
dbConfig.setAllowCreate(init);
|
||
dbConfig.setExclusiveCreate(init);
|
||
dbConfig.setTransactional(true);
|
||
/* JE-only begin */
|
||
db = env.openDatabase(null, "BigRandom", dbConfig);
|
||
/* JE-only end */
|
||
}
|
||
|
||
private void closeEnv()
|
||
throws DatabaseException {
|
||
|
||
db.close();
|
||
env.close();
|
||
}
|
||
|
||
public void run() {
|
||
|
||
/*
|
||
* The key is reused over multiple loop iterations for computing a
|
||
* local insertion key, so it must be instantiated at the top of the
|
||
* loop. In makeInsertKey a local insertion key is creating by adding
|
||
* one to the last key accessed.
|
||
*/
|
||
DatabaseEntry data = new DatabaseEntry();
|
||
DatabaseEntry key = new DatabaseEntry();
|
||
int localKeyTxns = 0;
|
||
byte[] lastInsertKey = null;
|
||
OperationStatus status;
|
||
|
||
while (!done) {
|
||
|
||
try {
|
||
/*
|
||
* When using local keys, only the first insert will be with a
|
||
* random key, and only if we've exceeded the maximum number of
|
||
* local key transactions. When not using local keys, all keys
|
||
* are randomly generated.
|
||
*/
|
||
boolean useLocalKeys = maxLocalKeyTxns > 0;
|
||
boolean insertRandomKey = true;
|
||
if (useLocalKeys) {
|
||
if (localKeyTxns < maxLocalKeyTxns) {
|
||
insertRandomKey = false;
|
||
localKeyTxns += 1;
|
||
} else {
|
||
localKeyTxns = 0;
|
||
}
|
||
}
|
||
|
||
/* Perform the transaction. */
|
||
for (int retry = 0;; retry += 1) {
|
||
Transaction txn = env.beginTransaction(null, null);
|
||
Cursor c = db.openCursor(txn, null);
|
||
try {
|
||
boolean addedKey = false;
|
||
if (init && nKeys < nMaxKeys) {
|
||
key.setData(lastInsertKey);
|
||
insert(c, key, data, insertRandomKey);
|
||
lastInsertKey = copyData(key);
|
||
insertRandomKey = !useLocalKeys;
|
||
addedKey = true;
|
||
}
|
||
if (!fastInit) {
|
||
/* Insert. */
|
||
key.setData(lastInsertKey);
|
||
insert(c, key, data, insertRandomKey);
|
||
lastInsertKey = copyData(key);
|
||
if (useLocalKeys) {
|
||
/* Update the following key. */
|
||
status = c.getNext(key, data, LockMode.RMW);
|
||
if (status == OperationStatus.SUCCESS) {
|
||
c.putCurrent(data);
|
||
/* Delete the following key. */
|
||
status = c.getNext
|
||
(key, data, LockMode.RMW);
|
||
if (status == OperationStatus.SUCCESS) {
|
||
c.delete();
|
||
}
|
||
}
|
||
/* Read. Use RMW to avoid deadlocks. */
|
||
for (int i = 0; i < nReadsPerWrite; i += 1) {
|
||
c.getNext(key, data, LockMode.RMW);
|
||
}
|
||
} else {
|
||
/* Update */
|
||
getRandom(c, key, data, LockMode.RMW);
|
||
c.putCurrent(data);
|
||
/* Delete */
|
||
getRandom(c, key, data, LockMode.RMW);
|
||
c.delete();
|
||
/* Read */
|
||
for (int i = 0; i < nReadsPerWrite; i += 1) {
|
||
getRandom(c, key, data, null);
|
||
}
|
||
}
|
||
}
|
||
c.close();
|
||
txn.commit();
|
||
nTransactions += 1;
|
||
if (addedKey) {
|
||
nKeys += 1;
|
||
}
|
||
if (initOnly && nKeys >= nMaxKeys) {
|
||
done = true;
|
||
}
|
||
if (nMaxTransactions != 0 &&
|
||
nTransactions >= nMaxTransactions) {
|
||
done = true;
|
||
}
|
||
break;
|
||
} catch (LockConflictException e) {
|
||
c.close();
|
||
txn.abort();
|
||
if (retry >= maxRetries) {
|
||
throw e;
|
||
}
|
||
/* Break deadlock cycle with a small sleep. */
|
||
Thread.sleep(5);
|
||
nDeadlocks += 1;
|
||
}
|
||
}
|
||
} catch (Throwable e) {
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
}
|
||
}
|
||
|
||
private void insert(Cursor c, DatabaseEntry key, DatabaseEntry data,
|
||
boolean insertRandomKey)
|
||
throws DatabaseException {
|
||
|
||
makeData(data);
|
||
while (true) {
|
||
makeInsertKey(c, key, insertRandomKey);
|
||
OperationStatus status = c.putNoOverwrite(key, data);
|
||
if (status == OperationStatus.KEYEXIST) {
|
||
if (sequentialKeys) {
|
||
System.out.println("****** Duplicate sequential key.");
|
||
} else if (insertRandomKey) {
|
||
System.out.println("****** Duplicate random key.");
|
||
} else {
|
||
System.out.println("****** Duplicate local key.");
|
||
}
|
||
} else {
|
||
if (status != OperationStatus.SUCCESS) {
|
||
System.out.println
|
||
("Unexpected return value from insert(): " + status);
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
private void getRandom(Cursor c, DatabaseEntry key, DatabaseEntry data,
|
||
LockMode lockMode)
|
||
throws DatabaseException {
|
||
|
||
makeRandomKey(key);
|
||
OperationStatus status = c.getSearchKeyRange(key, data, lockMode);
|
||
if (status == OperationStatus.NOTFOUND) {
|
||
status = c.getLast(key, data, lockMode);
|
||
if (status != OperationStatus.SUCCESS) {
|
||
System.out.println
|
||
("Unexpected return value from getRandom(): " + status);
|
||
}
|
||
}
|
||
}
|
||
|
||
private long getLastSequence()
|
||
throws DatabaseException {
|
||
|
||
if (!sequentialKeys) throw new IllegalStateException();
|
||
DatabaseEntry data = new DatabaseEntry();
|
||
DatabaseEntry key = new DatabaseEntry();
|
||
Cursor c = db.openCursor(null, null);
|
||
try {
|
||
OperationStatus status = c.getLast(key, data, null);
|
||
if (status == OperationStatus.SUCCESS) {
|
||
TupleInput in = new TupleInput(key.getData(),
|
||
key.getOffset(),
|
||
key.getSize());
|
||
return in.readLong();
|
||
} else {
|
||
return 0;
|
||
}
|
||
} finally {
|
||
c.close();
|
||
}
|
||
}
|
||
|
||
private void doPreload()
|
||
throws DatabaseException {
|
||
|
||
System.out.println("Preloading");
|
||
DatabaseEntry data = new DatabaseEntry();
|
||
DatabaseEntry key = new DatabaseEntry();
|
||
Cursor c = db.openCursor(null, null);
|
||
try {
|
||
long startTime = System.currentTimeMillis();
|
||
int count = 0;
|
||
while (c.getNext(key, data, LockMode.READ_UNCOMMITTED) ==
|
||
OperationStatus.SUCCESS) {
|
||
count += 1;
|
||
}
|
||
long endTime = System.currentTimeMillis();
|
||
int seconds = (int) ((endTime - startTime) / 1000);
|
||
System.out.println
|
||
("Preloaded records=" + count + " seconds=" + seconds);
|
||
} finally {
|
||
c.close();
|
||
}
|
||
}
|
||
|
||
private void makeInsertKey(Cursor c, DatabaseEntry key,
|
||
boolean insertRandomKey) {
|
||
if (sequentialKeys) {
|
||
long val;
|
||
synchronized (this) {
|
||
val = ++sequence;
|
||
}
|
||
makeLongKey(key, val);
|
||
} else if (!insertRandomKey && key.getData() != null) {
|
||
BigInteger num = new BigInteger(copyData(key));
|
||
num = num.add(BigInteger.ONE);
|
||
key.setData(num.toByteArray());
|
||
} else {
|
||
makeRandomKey(key);
|
||
}
|
||
}
|
||
|
||
private void makeRandomKey(DatabaseEntry key) {
|
||
|
||
if (sequentialKeys) {
|
||
makeLongKey(key, (long) (random.nextFloat() * sequence));
|
||
} else {
|
||
byte[] bytes = new byte[keySize];
|
||
random.nextBytes(bytes);
|
||
key.setData(bytes);
|
||
}
|
||
}
|
||
|
||
private void makeLongKey(DatabaseEntry key, long val) {
|
||
|
||
TupleOutput out = new TupleOutput();
|
||
out.writeLong(val);
|
||
byte[] pad = new byte[keySize - 8];
|
||
out.writeFast(pad);
|
||
if (out.getBufferOffset() != 0 || out.getBufferLength() != keySize) {
|
||
throw new IllegalStateException();
|
||
}
|
||
key.setData(out.getBufferBytes(), 0, keySize);
|
||
}
|
||
|
||
private void makeData(DatabaseEntry data) {
|
||
|
||
byte[] bytes = new byte[dataSize];
|
||
for (int i = 0; i < bytes.length; i += 1) {
|
||
bytes[i] = (byte) i;
|
||
}
|
||
data.setData(bytes);
|
||
}
|
||
|
||
private byte[] copyData(DatabaseEntry data) {
|
||
|
||
byte[] buf = new byte[data.getSize()];
|
||
System.arraycopy(data.getData(), data.getOffset(), buf, 0, buf.length);
|
||
return buf;
|
||
}
|
||
}
|