mirror of
https://github.com/berkeleydb/je.git
synced 2024-07-06 09:57:18 +00:00
529 lines
18 KiB
Java
529 lines
18 KiB
Java
/*-
|
||
* Copyright (C) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
|
||
*
|
||
* This file was distributed by Oracle as part of a version of Oracle Berkeley
|
||
* DB Java Edition made available at:
|
||
*
|
||
* http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html
|
||
*
|
||
* Please see the LICENSE file included in the top-level directory of the
|
||
* appropriate version of Oracle Berkeley DB Java Edition for a copy of the
|
||
* license and additional information.
|
||
*/
|
||
|
||
import java.io.File;
|
||
import java.text.NumberFormat;
|
||
import java.util.Random;
|
||
import java.util.concurrent.atomic.AtomicInteger;
|
||
|
||
import com.sleepycat.bind.tuple.TupleBinding;
|
||
import com.sleepycat.bind.tuple.TupleOutput;
|
||
import com.sleepycat.je.CheckpointConfig;
|
||
import com.sleepycat.je.Cursor;
|
||
import com.sleepycat.je.Database;
|
||
import com.sleepycat.je.DatabaseConfig;
|
||
import com.sleepycat.je.DatabaseEntry;
|
||
import com.sleepycat.je.DatabaseException;
|
||
import com.sleepycat.je.DbInternal;
|
||
import com.sleepycat.je.Environment;
|
||
import com.sleepycat.je.EnvironmentConfig;
|
||
import com.sleepycat.je.EnvironmentStats;
|
||
import com.sleepycat.je.StatsConfig;
|
||
|
||
/**
|
||
* Used to test a small cache and log cleaning. For example, to create a large
|
||
* set of log files (over 10 GB) that are almost 100% obsolete:
|
||
*
|
||
* java -Xmx6m -cp .:before.jar CleanWithSmallCache \
|
||
* -records 40000 -key 48 -data 10 -h tmp -random -cache 250k \
|
||
* -seconds 2000 -write 10000
|
||
*
|
||
* And then to clean that set of logs:
|
||
*
|
||
* java -Xmx15m -cp .:before.jar CleanWithSmallCache \
|
||
* -records 40000 -key 48 -data 10 -h tmp -random -cache 250k \
|
||
* -seconds 22000 -read 10 -clean
|
||
*/
|
||
public class CleanWithSmallCache {
|
||
|
||
private static final NumberFormat INT_FORMAT =
|
||
NumberFormat.getIntegerInstance();
|
||
private static final NumberFormat NUMBER_FORMAT =
|
||
NumberFormat.getNumberInstance();
|
||
|
||
private File envHome = null;
|
||
private int cacheSize = 0;
|
||
private boolean offHeap = false;
|
||
private int records = -1;
|
||
private int keySize = -1;
|
||
private int dataSize = -1;
|
||
private int fanout = 128;
|
||
private boolean doReads = false;
|
||
private boolean doWrites = false;
|
||
private int totalSeconds = 0;
|
||
private long beginTime = 0;
|
||
private long endTime = 0;
|
||
private boolean randomKeys = false;
|
||
private boolean doClean = false;
|
||
private boolean fillCache = false;
|
||
private Random random = new Random(123);
|
||
private AtomicInteger nReads = new AtomicInteger(0);
|
||
private AtomicInteger nWrites = new AtomicInteger(0);
|
||
private boolean programDone = false;
|
||
private Environment env = null;
|
||
private Database db = null;
|
||
|
||
public static void main(String[] args) {
|
||
try {
|
||
System.out.print("Command line: ");
|
||
for (String s : args) {
|
||
System.out.print(s);
|
||
System.out.print(' ');
|
||
}
|
||
System.out.println();
|
||
CleanWithSmallCache test = new CleanWithSmallCache(args);
|
||
long start = System.currentTimeMillis();
|
||
System.out.println("Opening environment");
|
||
test.open();
|
||
System.out.println("Starting test");
|
||
test.execute();
|
||
test.close();
|
||
long end = System.currentTimeMillis();
|
||
System.out.println("Time: " + ((end - start) / 1000) + " sec");
|
||
System.exit(0);
|
||
} catch (Throwable e) {
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
}
|
||
|
||
private CleanWithSmallCache(String[] args) {
|
||
|
||
for (int i = 0; i < args.length; i += 1) {
|
||
String name = args[i];
|
||
String val = null;
|
||
if (i < args.length - 1 && !args[i + 1].startsWith("-")) {
|
||
i += 1;
|
||
val = args[i];
|
||
}
|
||
if (name.equals("-h")) {
|
||
if (val == null) {
|
||
usage("No value after -h");
|
||
}
|
||
envHome = new File(val);
|
||
} else if (name.equals("-cache")) {
|
||
if (val == null) {
|
||
usage("No value after -cache");
|
||
}
|
||
boolean mb = false;
|
||
boolean kb = false;
|
||
if (val.endsWith("m")) {
|
||
mb = true;
|
||
val = val.substring(0, val.length() - 1);
|
||
} else if (val.endsWith("k")) {
|
||
kb = true;
|
||
val = val.substring(0, val.length() - 1);
|
||
}
|
||
try {
|
||
cacheSize = Integer.parseInt(val);
|
||
} catch (NumberFormatException e) {
|
||
usage(val + " is not a number");
|
||
}
|
||
if (cacheSize <= 0) {
|
||
usage(val + " is not a positive integer");
|
||
}
|
||
if (mb) {
|
||
cacheSize *= 1024 * 1024;
|
||
} else if (kb) {
|
||
cacheSize *= 1024;
|
||
}
|
||
} else if (name.equals("-offheap")) {
|
||
if (val == null) {
|
||
usage("No value after -offheap");
|
||
}
|
||
offHeap = Boolean.parseBoolean(val);
|
||
} else if (name.equals("-records")) {
|
||
if (val == null) {
|
||
usage("No value after -records");
|
||
}
|
||
try {
|
||
records = Integer.parseInt(val);
|
||
} catch (NumberFormatException e) {
|
||
usage(val + " is not a number");
|
||
}
|
||
if (records <= 0) {
|
||
usage(val + " is not a positive integer");
|
||
}
|
||
} else if (name.equals("-key")) {
|
||
if (val == null) {
|
||
usage("No value after -key");
|
||
}
|
||
try {
|
||
keySize = Integer.parseInt(val);
|
||
} catch (NumberFormatException e) {
|
||
usage(val + " is not a number");
|
||
}
|
||
if (keySize <= 0) {
|
||
usage(val + " is not a positive integer");
|
||
}
|
||
} else if (name.equals("-data")) {
|
||
if (val == null) {
|
||
usage("No value after -data");
|
||
}
|
||
try {
|
||
dataSize = Integer.parseInt(val);
|
||
} catch (NumberFormatException e) {
|
||
usage(val + " is not a number");
|
||
}
|
||
if (dataSize < 0) {
|
||
usage(val + " is not a non-negative integer");
|
||
}
|
||
} else if (name.equals("-read")) {
|
||
if (val == null) {
|
||
usage("No value after -read");
|
||
}
|
||
doReads = Boolean.parseBoolean(val);
|
||
} else if (name.equals("-write")) {
|
||
if (val == null) {
|
||
usage("No value after -write");
|
||
}
|
||
doWrites = Boolean.parseBoolean(val);
|
||
} else if (name.equals("-seconds")) {
|
||
if (val == null) {
|
||
usage("No value after -seconds");
|
||
}
|
||
try {
|
||
totalSeconds = Integer.parseInt(val);
|
||
} catch (NumberFormatException e) {
|
||
usage(val + " is not a number");
|
||
}
|
||
if (totalSeconds < 0) {
|
||
usage(val + " is not a non-negative integer");
|
||
}
|
||
} else if (name.equals("-fanout")) {
|
||
if (val == null) {
|
||
usage("No value after -fanout");
|
||
}
|
||
try {
|
||
fanout = Integer.parseInt(val);
|
||
} catch (NumberFormatException e) {
|
||
usage(val + " is not a number");
|
||
}
|
||
if (fanout <= 0) {
|
||
usage(val + " is not a positive integer");
|
||
}
|
||
} else if (name.equals("-random")) {
|
||
randomKeys = true;
|
||
} else if (name.equals("-clean")) {
|
||
doClean = true;
|
||
} else if (name.equals("-fillcache")) {
|
||
fillCache = true;
|
||
} else {
|
||
usage("Unknown arg: " + name);
|
||
}
|
||
}
|
||
|
||
if (envHome == null) {
|
||
usage("-h not specified");
|
||
}
|
||
|
||
if (cacheSize <= 0) {
|
||
usage("-cache not specified");
|
||
}
|
||
|
||
if (records <= 0) {
|
||
usage("-records not specified");
|
||
}
|
||
|
||
if (keySize <= 0) {
|
||
usage("-key not specified");
|
||
}
|
||
|
||
if (dataSize <= 0) {
|
||
usage("-data not specified");
|
||
}
|
||
|
||
int maxRecNum;
|
||
switch (keySize) {
|
||
case 1:
|
||
maxRecNum = Byte.MAX_VALUE;
|
||
break;
|
||
case 2:
|
||
case 3:
|
||
maxRecNum = Short.MAX_VALUE;
|
||
break;
|
||
default:
|
||
maxRecNum = Integer.MAX_VALUE;
|
||
}
|
||
if (records > maxRecNum) {
|
||
usage("-key size too small for number of records");
|
||
}
|
||
}
|
||
|
||
private void usage(String msg) {
|
||
|
||
if (msg != null) {
|
||
System.out.println(msg);
|
||
}
|
||
|
||
System.out.println
|
||
("usage:" +
|
||
"\njava " + CleanWithSmallCache.class.getName() +
|
||
"\n -h <envHome>" +
|
||
"\n # Environment home directory" +
|
||
"\n -records <count>" +
|
||
"\n # Total records (key/data pairs); required" +
|
||
"\n -key <bytes>" +
|
||
"\n # Key bytes per record; required" +
|
||
"\n -data <bytes>" +
|
||
"\n # Data bytes per record; required" +
|
||
"\n [-fanout <entries>]" +
|
||
"\n # Number of entries per Btree node; default: 128" +
|
||
"\n [-read <readsPerSecond>]" +
|
||
"\n # Number of read operations per second; default: 0" +
|
||
"\n [-write <writesPerSecond>]" +
|
||
"\n # Number of write operations per second; default: 0" +
|
||
"\n [-random]" +
|
||
"\n # Write randomly generated keys;" +
|
||
"\n # default: write sequential keys" +
|
||
"\n [-seconds <totalSeconds>]" +
|
||
"\n # Number of seconds to run; default: 0 or forever" +
|
||
"\n [-clean]" +
|
||
"\n # Perform log cleaning; default: false" +
|
||
"\n [-offheap]" +
|
||
"\n # Use an off-heap cache; default: false" +
|
||
"\n [-fillcache]" +
|
||
"\n # Artificially fill the cache; default: false");
|
||
|
||
System.exit(2);
|
||
}
|
||
|
||
private void open()
|
||
throws DatabaseException {
|
||
|
||
EnvironmentConfig envConfig = new EnvironmentConfig();
|
||
envConfig.setAllowCreate(true);
|
||
envConfig.setConfigParam("je.env.runCleaner", "false");
|
||
envConfig.setCacheSize(cacheSize);
|
||
if (offHeap) {
|
||
envConfig.setOffHeapCacheSize(cacheSize);
|
||
envConfig.setConfigParam(
|
||
EnvironmentConfig.OFFHEAP_EVICT_BYTES, "1024");
|
||
}
|
||
env = new Environment(envHome, envConfig);
|
||
|
||
DatabaseConfig dbConfig = new DatabaseConfig();
|
||
dbConfig.setAllowCreate(true);
|
||
dbConfig.setNodeMaxEntries(fanout);
|
||
db = env.openDatabase(null, "foo", dbConfig);
|
||
|
||
if (fillCache) {
|
||
DbInternal.getNonNullEnvImpl(env).getMemoryBudget().
|
||
updateAdminMemoryUsage(cacheSize * 2);
|
||
}
|
||
}
|
||
|
||
private void close()
|
||
throws DatabaseException {
|
||
|
||
db.close();
|
||
env.close();
|
||
}
|
||
|
||
private int makeKey(int recNum, DatabaseEntry entry) {
|
||
if (randomKeys) {
|
||
recNum = random.nextInt(records - 1) + 1;
|
||
} else {
|
||
recNum += 1;
|
||
if (recNum > records) {
|
||
recNum = 1;
|
||
}
|
||
}
|
||
if (keySize == 1) {
|
||
entry.setData(new byte[] { (byte) recNum });
|
||
} else {
|
||
TupleOutput out = new TupleOutput(new byte[keySize]);
|
||
int written;
|
||
if (keySize == 2 || keySize == 3) {
|
||
out.writeUnsignedShort((short) recNum);
|
||
written = 2;
|
||
} else {
|
||
out.writeUnsignedInt(recNum);
|
||
written = 4;
|
||
}
|
||
while (written < keySize) {
|
||
out.writeFast(0);
|
||
written += 1;
|
||
}
|
||
TupleBinding.outputToEntry(out, entry);
|
||
}
|
||
return recNum;
|
||
}
|
||
|
||
private void execute()
|
||
throws InterruptedException {
|
||
|
||
Thread monitor = new Monitor();
|
||
Thread cleaner = null;
|
||
if (doClean) {
|
||
cleaner = new Cleaner();
|
||
}
|
||
Thread writer = null;
|
||
if (doWrites) {
|
||
writer = new OperationRunner(nWrites, new Operation() {
|
||
public void doOperation(DatabaseEntry key, DatabaseEntry data)
|
||
throws DatabaseException {
|
||
db.put(null, key, data);
|
||
}
|
||
});
|
||
}
|
||
Thread reader = null;
|
||
if (doReads) {
|
||
reader = new OperationRunner(nReads, new Operation() {
|
||
public void doOperation(DatabaseEntry key, DatabaseEntry data)
|
||
throws DatabaseException {
|
||
Cursor cursor = db.openCursor(null, null);
|
||
cursor.getSearchKeyRange(key, data, null);
|
||
cursor.close();
|
||
}
|
||
});
|
||
}
|
||
beginTime = System.currentTimeMillis();
|
||
if (totalSeconds > 0) {
|
||
endTime = beginTime + (totalSeconds * 1000);
|
||
}
|
||
monitor.start();
|
||
if (cleaner != null) {
|
||
cleaner.start();
|
||
}
|
||
if (writer != null) {
|
||
writer.start();
|
||
}
|
||
if (reader != null) {
|
||
reader.start();
|
||
}
|
||
monitor.join();
|
||
if (cleaner != null) {
|
||
cleaner.join();
|
||
}
|
||
if (writer != null) {
|
||
writer.join();
|
||
}
|
||
if (reader != null) {
|
||
reader.join();
|
||
}
|
||
}
|
||
|
||
private class Monitor extends Thread {
|
||
public void run() {
|
||
try {
|
||
long lastTime = System.currentTimeMillis();
|
||
while ((totalSeconds == 0 || lastTime < endTime) &&
|
||
!programDone) {
|
||
Thread.sleep(5000);
|
||
long time = System.currentTimeMillis();
|
||
printStats(time);
|
||
lastTime = time;
|
||
}
|
||
programDone = true;
|
||
} catch (Throwable e) {
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
}
|
||
}
|
||
|
||
private class Cleaner extends Thread {
|
||
public void run() {
|
||
CheckpointConfig forceConfig = new CheckpointConfig();
|
||
forceConfig.setForce(true);
|
||
try {
|
||
boolean cleanedSome;
|
||
do {
|
||
cleanedSome = false;
|
||
while (true) {
|
||
int nFiles = env.cleanLog();
|
||
if (nFiles == 0) {
|
||
break;
|
||
}
|
||
cleanedSome = true;
|
||
}
|
||
env.checkpoint(forceConfig);
|
||
} while (cleanedSome && !programDone);
|
||
programDone = true;
|
||
} catch (Throwable e) {
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
}
|
||
}
|
||
|
||
private interface Operation {
|
||
void doOperation(DatabaseEntry key, DatabaseEntry data)
|
||
throws DatabaseException;
|
||
}
|
||
|
||
private class OperationRunner extends Thread {
|
||
|
||
private Operation op;
|
||
private AtomicInteger nOps;
|
||
|
||
OperationRunner(AtomicInteger nOps, Operation op) {
|
||
this.nOps = nOps;
|
||
this.op = op;
|
||
}
|
||
|
||
public void run() {
|
||
|
||
int recNum = 0;
|
||
int ops = 0;
|
||
long beforeTime = System.currentTimeMillis();
|
||
DatabaseEntry key = new DatabaseEntry();
|
||
DatabaseEntry data = new DatabaseEntry(new byte[dataSize]);
|
||
|
||
try {
|
||
while (!programDone) {
|
||
recNum = makeKey(recNum, key);
|
||
op.doOperation(key, data);
|
||
ops += 1;
|
||
nOps.incrementAndGet();
|
||
}
|
||
} catch (Throwable e) {
|
||
e.printStackTrace();
|
||
System.exit(1);
|
||
}
|
||
}
|
||
}
|
||
|
||
private void printStats(long currentTime)
|
||
throws DatabaseException {
|
||
|
||
StatsConfig statsConfig = new StatsConfig();
|
||
statsConfig.setClear(true);
|
||
EnvironmentStats stats = env.getStats(statsConfig);
|
||
|
||
float secs = (currentTime - beginTime) / 1000.0f;
|
||
float writesPerSec = nWrites.get() / secs;
|
||
float readsPerSec = nReads.get() / secs;
|
||
|
||
System.out.println("\nWrites/Sec=" +
|
||
NUMBER_FORMAT.format(writesPerSec) +
|
||
" Reads/Sec=" +
|
||
NUMBER_FORMAT.format(readsPerSec) +
|
||
" CacheSize=" +
|
||
INT_FORMAT.format(stats.getCacheTotalBytes()) +
|
||
" DataSize=" +
|
||
INT_FORMAT.format(stats.getDataBytes()) +
|
||
" AdminSize=" +
|
||
INT_FORMAT.format(stats.getAdminBytes()) +
|
||
" LockSize=" +
|
||
INT_FORMAT.format(stats.getLockBytes()) +
|
||
" NEvictPasses=" +
|
||
INT_FORMAT.format(stats.getNEvictPasses()) +
|
||
" NCacheMiss=" +
|
||
INT_FORMAT.format(stats.getNCacheMiss()) +
|
||
" TotalLogSize=" +
|
||
INT_FORMAT.format(stats.getTotalLogSize()));
|
||
}
|
||
}
|