/*- * See the file LICENSE for redistribution information. * * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * */ package repmgrtests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.net.Socket; import org.junit.After; import org.junit.Before; import org.junit.Test; import com.sleepycat.db.Database; import com.sleepycat.db.DatabaseConfig; import com.sleepycat.db.DatabaseEntry; import com.sleepycat.db.DatabaseType; import com.sleepycat.db.Environment; import com.sleepycat.db.EnvironmentConfig; import com.sleepycat.db.EventHandlerAdapter; import com.sleepycat.db.ReplicationManagerSiteConfig; import com.sleepycat.db.ReplicationManagerStartPolicy; import com.sleepycat.db.ReplicationManagerStats; import com.sleepycat.db.ReplicationTimeoutType; import com.sleepycat.db.StatsConfig; import com.sleepycat.db.VerboseConfig; /** * Tests for repmgr heartbeat feature. */ public class TestHeartbeats { private static final String TEST_DIR_NAME = "TESTDIR"; private File testdir; private Process fiddler; Socket s; private int masterPort; private int clientPort; private int client2Port; private int mgrPort; class MyEventHandler extends EventHandlerAdapter { private boolean done = false; private boolean panic = false; private int newmasterCount = 0; private boolean iAmMaster; @Override synchronized public void handleRepStartupDoneEvent() { done = true; notifyAll(); } @Override synchronized public void handleRepMasterEvent() { iAmMaster = true; } @Override synchronized public void handleRepNewMasterEvent(int eid) { newmasterCount++; } synchronized public int getNewmasterCount() { return newmasterCount; } synchronized public boolean isMaster() { return iAmMaster; } @Override synchronized public void handlePanicEvent() { done = true; panic = true; notifyAll(); } synchronized void await() throws Exception { while (!done) { wait(); } if (panic) throw new Exception("aborted by panic in DB"); } } @Before public void setUp() throws Exception { testdir = new File(TEST_DIR_NAME); Util.rm_rf(testdir); testdir.mkdir(); if (Boolean.getBoolean("MANUAL_FIDDLER_START")) { masterPort = 6000; clientPort = 6001; client2Port = 6002; mgrPort = 8000; fiddler = null; } else { String mgrPortNum = System.getenv("DB_TEST_FAKE_PORT"); assertNotNull("required DB_TEST_FAKE_PORT environment variable not found", mgrPortNum); mgrPort = Integer.parseInt(mgrPortNum); PortsConfig p = new PortsConfig(3); masterPort = p.getRealPort(0); clientPort = p.getRealPort(1); client2Port = p.getRealPort(2); fiddler = Util.startFiddler(p, getClass().getName(), mgrPort); } } @After public void tearDown() throws Exception { if (fiddler != null) { fiddler.destroy(); } } // (Is 2 minutes enough?) // @Test(timeout=120000) public void testMonitorCallElect() throws Exception { EnvironmentConfig masterConfig = makeBasicConfig(); ReplicationManagerSiteConfig site = new ReplicationManagerSiteConfig("localhost", masterPort); site.setLocalSite(true); masterConfig.addReplicationManagerSite(site); MyEventHandler masterMonitor = new MyEventHandler(); masterConfig.setEventHandler(masterMonitor); File masterDir = mkdir("master"); Environment master = new Environment(masterDir, masterConfig); master.setReplicationTimeout(ReplicationTimeoutType.HEARTBEAT_SEND, 3000000); master.setReplicationTimeout(ReplicationTimeoutType.HEARTBEAT_MONITOR, 5000000); master.setReplicationTimeout(ReplicationTimeoutType.CONNECTION_RETRY, 3000000); master.replicationManagerStart(3, ReplicationManagerStartPolicy.REP_MASTER); DatabaseConfig dc = new DatabaseConfig(); dc.setTransactional(true); dc.setAllowCreate(true); dc.setType(DatabaseType.BTREE); Database db = master.openDatabase(null, "test.db", null, dc); // create two clients, wait for them to finish sync-ing up // MyEventHandler clientMonitor = new MyEventHandler(); EnvironmentConfig ec = makeClientConfig(clientMonitor, clientPort, masterPort); File clientDir = mkdir("client"); Environment client = new Environment(clientDir, ec); client.setReplicationTimeout(ReplicationTimeoutType.HEARTBEAT_SEND, 3000000); client.setReplicationTimeout(ReplicationTimeoutType.HEARTBEAT_MONITOR, 5000000); client.setReplicationTimeout(ReplicationTimeoutType.CONNECTION_RETRY, 3000000); client.replicationManagerStart(1, ReplicationManagerStartPolicy.REP_CLIENT); clientMonitor.await(); MyEventHandler client2Monitor = new MyEventHandler(); ec = makeClientConfig(client2Monitor, client2Port, masterPort); File client2Dir = mkdir("client2"); Environment client2 = new Environment(client2Dir, ec); client2.setReplicationTimeout(ReplicationTimeoutType.HEARTBEAT_SEND, 3000000); client2.setReplicationTimeout(ReplicationTimeoutType.HEARTBEAT_MONITOR, 5000000); client2.setReplicationTimeout(ReplicationTimeoutType.CONNECTION_RETRY, 3000000); client2.replicationManagerStart(1, ReplicationManagerStartPolicy.REP_CLIENT); client2Monitor.await(); StatsConfig statsConfig = new StatsConfig(); statsConfig.setClear(true); master.getReplicationManagerStats(statsConfig); // do some transactions, at a leisurely pace (but quicker than // the hb period) // DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); value.setData("foo".getBytes()); for (int i=0; i<3; i++) { Thread.sleep(2000); String k = "The record number is: " + i; key.setData(k.getBytes()); db.put(null, key, value); } // a few more, but not quick enough to prevent HB's // for (int i=3; i<6; i++) { Thread.sleep(4000); String k = "The record number is: " + i; key.setData(k.getBytes()); db.put(null, key, value); } // pause for a few HB times Thread.sleep(10000); assertFalse(clientMonitor.isMaster() || client2Monitor.isMaster()); // tell fiddler to freeze (not really the right word) the // connection paths from master to both clients. // if (s == null) s = new Socket("localhost", mgrPort); OutputStreamWriter w = new OutputStreamWriter(s.getOutputStream()); String path1 = "{" + masterPort + "," + clientPort + "}"; // looks like {6000,6001} w.write("{" + path1 + ",toss_all}\r\n"); w.flush(); BufferedReader br = new BufferedReader(new InputStreamReader(s.getInputStream())); assertEquals("ok", br.readLine()); String path2 = "{" + masterPort + "," + client2Port + "}"; // looks like {6000,6002} w.write("{" + path2 + ",toss_all}\r\n"); w.flush(); assertEquals("ok", br.readLine()); // wait for the clients to notice a problem (5 sec max), plus // give them a couple extra seconds to hold an election // Thread.sleep(5000 + 2000); ReplicationManagerStats masterStats = master.getReplicationManagerStats(StatsConfig.DEFAULT); assertEquals(2, masterStats.getConnectionDrop()); assertTrue(clientMonitor.isMaster() || client2Monitor.isMaster()); client2.close(); client.close(); db.close(); master.close(); w.write("shutdown\r\n"); w.flush(); assertEquals("ok", br.readLine()); s.close(); fiddler = null; } private EnvironmentConfig makeClientConfig(MyEventHandler evHandler, int clientPort, int masterPort) throws Exception { EnvironmentConfig ec = makeBasicConfig(); ReplicationManagerSiteConfig site = new ReplicationManagerSiteConfig("localhost", clientPort); site.setLocalSite(true); ec.addReplicationManagerSite(site); site = new ReplicationManagerSiteConfig("localhost", masterPort); site.setBootstrapHelper(true); ec.addReplicationManagerSite(site); ec.setEventHandler(evHandler); return (ec); } public static EnvironmentConfig makeBasicConfig() { EnvironmentConfig ec = new EnvironmentConfig(); ec.setAllowCreate(true); ec.setInitializeCache(true); ec.setInitializeLocking(true); ec.setInitializeLogging(true); ec.setInitializeReplication(true); ec.setTransactional(true); ec.setThreaded(true); if (Boolean.getBoolean("VERB_REPLICATION")) ec.setVerbose(VerboseConfig.REPLICATION, true); return (ec); } public File mkdir(String dname) { File f = new File(testdir, dname); f.mkdir(); return f; } }