mirror of
https://github.com/berkeleydb/libdb.git
synced 2024-11-16 17:16:25 +00:00
658 lines
25 KiB
HTML
658 lines
25 KiB
HTML
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||
<head>
|
||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||
<title>Example Processing Loop</title>
|
||
<link rel="stylesheet" href="gettingStarted.css" type="text/css" />
|
||
<meta name="generator" content="DocBook XSL Stylesheets V1.73.2" />
|
||
<link rel="start" href="index.html" title="Getting Started with Replicated Berkeley DB Applications" />
|
||
<link rel="up" href="fwrkmasterreplica.html" title="Chapter 4. Replica versus Master Processes" />
|
||
<link rel="prev" href="processingloop.html" title="Processing Loop" />
|
||
<link rel="next" href="addfeatures.html" title="Chapter 5. Additional Features" />
|
||
</head>
|
||
<body>
|
||
<div xmlns="" class="navheader">
|
||
<div class="libver">
|
||
<p>Library Version 11.2.5.3</p>
|
||
</div>
|
||
<table width="100%" summary="Navigation header">
|
||
<tr>
|
||
<th colspan="3" align="center">Example Processing Loop</th>
|
||
</tr>
|
||
<tr>
|
||
<td width="20%" align="left"><a accesskey="p" href="processingloop.html">Prev</a> </td>
|
||
<th width="60%" align="center">Chapter 4. Replica versus Master Processes</th>
|
||
<td width="20%" align="right"> <a accesskey="n" href="addfeatures.html">Next</a></td>
|
||
</tr>
|
||
</table>
|
||
<hr />
|
||
</div>
|
||
<div class="sect1" lang="en" xml:lang="en">
|
||
<div class="titlepage">
|
||
<div>
|
||
<div>
|
||
<h2 class="title" style="clear: both"><a id="exampledoloop"></a>Example Processing Loop</h2>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="toc">
|
||
<dl>
|
||
<dt>
|
||
<span class="sect2">
|
||
<a href="exampledoloop.html#runningit">Running It</a>
|
||
</span>
|
||
</dt>
|
||
</dl>
|
||
</div>
|
||
<p>
|
||
In this section we take the example
|
||
processing loop that we presented in the
|
||
previous section and we flesh it out to
|
||
provide a more complete example. We do this
|
||
by updating the
|
||
<code class="function">doloop()</code>
|
||
function that our original transaction
|
||
application used
|
||
|
||
|
||
<span>(see <a class="xref" href="simpleprogramlisting.html#doloop_java" title="Method: SimpleTxn.doloop()">Method: SimpleTxn.doloop()</a>)</span>
|
||
to fully support our replicated application.
|
||
</p>
|
||
<p>
|
||
In the following example code, code that we
|
||
add to the original example is presented in
|
||
<strong class="userinput"><code>bold</code></strong>.
|
||
</p>
|
||
<p>
|
||
To begin, we must implement a way to track whether our
|
||
application is running as a master or a client. There are many
|
||
ways to do this, but in this case what we will do is extend
|
||
<code class="classname">com.sleepycat.db.Environment</code> to carry
|
||
the information. We do this by creating the
|
||
<code class="classname">RepQuoteEnvironment</code> class.
|
||
</p>
|
||
<pre class="programlisting">package db.repquote;
|
||
|
||
import com.sleepycat.db.DatabaseException;
|
||
import com.sleepycat.db.Environment;
|
||
import com.sleepycat.db.EnvironmentConfig;
|
||
|
||
public class RepQuoteEnvironment extends Environment
|
||
{
|
||
private boolean isMaster;
|
||
|
||
public RepQuoteEnvironment(final java.io.File host,
|
||
EnvironmentConfig config)
|
||
throws DatabaseException, java.io.FileNotFoundException
|
||
{
|
||
super(host, config);
|
||
isMaster = false;
|
||
}
|
||
|
||
boolean getIsMaster()
|
||
{
|
||
return isMaster;
|
||
}
|
||
|
||
public void setIsMaster(boolean isMaster)
|
||
{
|
||
this.isMaster = isMaster;
|
||
}
|
||
}</pre>
|
||
<p>
|
||
Next, we go to <code class="filename">RepQuoteExampleGSG.java</code> and
|
||
we include the
|
||
<code class="classname">RepQuoteEnvironment</code> class
|
||
as well as the
|
||
<code class="classname">EventHandler</code> class. We then
|
||
cause our <code class="classname">RepQuoteExampleGSG</code>
|
||
class to implement
|
||
<code class="classname">EventHandler</code>. We also
|
||
change our environment handle to be an instance
|
||
of <code class="classname">RepQuoteEnvironment</code>
|
||
instead of <code class="classname">Environment</code>.
|
||
</p>
|
||
<p>
|
||
Note that we also import the
|
||
<code class="classname">com.sleepycat.db.ReplicationHandleDeadException</code>
|
||
class. We will discuss what that exception is
|
||
used for a little later in this example.
|
||
</p>
|
||
<pre class="programlisting">package db.repquote;
|
||
|
||
import java.io.FileNotFoundException;
|
||
import java.io.BufferedReader;
|
||
import java.io.InputStreamReader;
|
||
import java.io.IOException;
|
||
import java.io.UnsupportedEncodingException;
|
||
import java.lang.Thread;
|
||
import java.lang.InterruptedException;
|
||
|
||
import com.sleepycat.db.Cursor;
|
||
import com.sleepycat.db.Database;
|
||
import com.sleepycat.db.DatabaseConfig;
|
||
import com.sleepycat.db.DatabaseEntry;
|
||
import com.sleepycat.db.DatabaseException;
|
||
import com.sleepycat.db.DatabaseType;
|
||
import com.sleepycat.db.EnvironmentConfig;
|
||
<strong class="userinput"><code>import com.sleepycat.db.EventHandler;</code></strong>
|
||
import com.sleepycat.db.LockMode;
|
||
import com.sleepycat.db.OperationStatus;
|
||
<strong class="userinput"><code>import com.sleepycat.db.ReplicationHandleDeadException;</code></strong>
|
||
import com.sleepycat.db.ReplicationHostAddress;
|
||
import com.sleepycat.db.ReplicationManagerSiteConfig;
|
||
|
||
import db.repquote.RepConfig;
|
||
<strong class="userinput"><code>import db.repquote.RepQuoteEnvironment</code></strong>
|
||
|
||
public class RepQuoteExampleGSG <strong class="userinput"><code> implements EventHandler</code></strong>
|
||
{
|
||
private RepConfig repConfig;
|
||
private <strong class="userinput"><code>RepQuoteEnvironment</code></strong> dbenv; </pre>
|
||
<p>
|
||
That done, we can skip the
|
||
|
||
|
||
<span><code class="methodname">main()</code> method and
|
||
our class constructor, because they do not change.</span>
|
||
|
||
Instead, we skip down to our
|
||
|
||
|
||
<span><code class="methodname">init()</code> method
|
||
where we take care of opening our environment and setting
|
||
the event handler. </span>
|
||
</p>
|
||
<p>
|
||
To update our <code class="methodname">init()</code> method, we only need
|
||
to do a couple of things. First, we identify the current class as
|
||
the event handler. Then, when we open our environment, we
|
||
instantiate a <code class="classname">RepQuoteEnvironment</code>
|
||
class instead of an <code class="classname">Environment</code>
|
||
class.
|
||
</p>
|
||
<pre class="programlisting"> public int init(RepConfig config)
|
||
throws DatabaseException
|
||
{
|
||
int ret = 0;
|
||
repConfig = config;
|
||
EnvironmentConfig envConfig = new EnvironmentConfig();
|
||
envConfig.setErrorStream(System.err);
|
||
envConfig.setErrorPrefix(RepConfig.progname);
|
||
|
||
envConfig.addReplicationManagerSite(repConfig.getThisHost());
|
||
for (ReplicationHostAddress host =
|
||
repConfig.getFirstOtherHost(); host != null;
|
||
host = repConfig.getNextOtherHost()){
|
||
|
||
ReplicationManagerSiteConfig repmgrRemoteSiteConfig =
|
||
new ReplicationManagerSiteConfig(host.host, host.port);
|
||
repmgrRemoteSiteConfig.setBootstrapHelper(true);
|
||
envConfig.addReplicationManagerSite(
|
||
repmgrRemoteSiteConfig);
|
||
}
|
||
|
||
envConfig.setReplicationPriority(repConfig.priority);
|
||
|
||
envConfig.setReplicationManagerAckPolicy(
|
||
ReplicationManagerAckPolicy.ALL);
|
||
envConfig.setCacheSize(RepConfig.CACHESIZE);
|
||
envConfig.setTxnNoSync(true);
|
||
|
||
<strong class="userinput"><code>envConfig.setEventHandler(this);</code></strong>
|
||
|
||
envConfig.setAllowCreate(true);
|
||
envConfig.setRunRecovery(true);
|
||
envConfig.setThreaded(true);
|
||
envConfig.setInitializeReplication(true);
|
||
envConfig.setInitializeLocking(true);
|
||
envConfig.setInitializeLogging(true);
|
||
envConfig.setInitializeCache(true);
|
||
envConfig.setTransactional(true);
|
||
envConfig.setVerboseReplication(appConfig.verbose);
|
||
try {
|
||
dbenv = new <strong class="userinput"><code>RepQuoteEnvironment</code></strong>(repConfig.getHome(),
|
||
envConfig);
|
||
} catch(FileNotFoundException e) {
|
||
System.err.println("FileNotFound exception: " + e.toString());
|
||
System.err.println(
|
||
"Ensure that the environment directory is pre-created.");
|
||
ret = 1;
|
||
}
|
||
|
||
// start Replication Manager
|
||
dbenv.replicationManagerStart(3, repConfig.startPolicy);
|
||
return ret;
|
||
} </pre>
|
||
<p>
|
||
That done, we need to implement the methods
|
||
required for responding to replication events.
|
||
These methods are required because we are now implementing
|
||
<code class="classname">com.sleepycat.db.EventHandler</code>. While
|
||
we are required to provide an implementation for all of these
|
||
methods, for our simple application we are really only interested
|
||
in these because they allow us to track whether a replication
|
||
instance is a master.
|
||
</p>
|
||
<pre class="programlisting"> public void handleRepClientEvent()
|
||
{
|
||
dbenv.setIsMaster(false);
|
||
}
|
||
|
||
public void handleRepConnectBrokenEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepConnectEstablishedEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepConnectTryFailedEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepMasterEvent()
|
||
{
|
||
dbenv.setIsMaster(true);
|
||
}
|
||
|
||
public void handleRepNewMasterEvent(int envId)
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleWriteFailedEvent(int errorCode)
|
||
{
|
||
System.err.println("Write to stable storage failed!" +
|
||
"Operating system error code:" + errorCode);
|
||
System.err.println("Continuing....");
|
||
}
|
||
|
||
public void handleRepStartupDoneEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepPermFailedEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepLocalSiteRemovedEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepSiteAddedEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepSiteRemovedEvent()
|
||
{
|
||
// Ignored for now.
|
||
}
|
||
|
||
public void handleRepElectedEvent()
|
||
{
|
||
// Safely ignored for Replication Manager applications.
|
||
}
|
||
|
||
public void handleRepElectionFailedEvent()
|
||
{
|
||
// Safely ignored for Replication Manager applications that do
|
||
// not manage their own master selection.
|
||
}
|
||
|
||
public void handleRepJoinFailureEvent()
|
||
{
|
||
// Safely ignored since this application did not turn off AUTOINIT.
|
||
}
|
||
|
||
public void handleRepMasterFailureEvent()
|
||
{
|
||
// Safely ignored for Replication Manager applications that do
|
||
// not manage their own master selection.
|
||
}
|
||
|
||
public void handleRepDupmasterEvent()
|
||
{
|
||
// Safely ignored for Replication Manager applications that do
|
||
// not manage their own master selection.
|
||
}
|
||
|
||
public void handlePanicEvent()
|
||
{
|
||
System.err.println("Panic encountered!");
|
||
System.err.println("Shutting down.");
|
||
System.err.println("You should restart, running recovery.");
|
||
try {
|
||
terminate();
|
||
} catch (DatabaseException dbe) {
|
||
System.err.println("Caught an exception during " +
|
||
"termination in handlePanicEvent: " + dbe.toString());
|
||
}
|
||
System.exit(-1);
|
||
} </pre>
|
||
<p>
|
||
That done, we need to update our
|
||
<code class="function">doloop()</code>
|
||
|
||
|
||
<span>method.</span>
|
||
</p>
|
||
<p>
|
||
We begin by updating our <code class="classname">DatabaseConfig</code>
|
||
instance to determine which options to use, depending on whether the
|
||
application is running as a master.
|
||
</p>
|
||
<pre class="programlisting"> public int doloop()
|
||
throws DatabaseException
|
||
{
|
||
Database db = null;
|
||
|
||
for (;;)
|
||
{
|
||
if (db == null) {
|
||
DatabaseConfig dbconf = new DatabaseConfig();
|
||
dbconf.setType(DatabaseType.BTREE);
|
||
<strong class="userinput"><code>if (dbenv.getIsMaster()) {
|
||
dbconf.setAllowCreate(true);
|
||
}</code></strong>
|
||
dbconf.setTransactional(true); </pre>
|
||
<p>
|
||
When we open the database, we modify our error handling to
|
||
account for the case where the database does not yet exist. This can
|
||
happen if our code is running as a replica and the Replication Manager has not
|
||
yet had a chance to create the databases for us. Recall that replicas never
|
||
write to their own databases directly, and so they cannot
|
||
create databases on their own.
|
||
</p>
|
||
<p>
|
||
If we detect that the database does not yet exist, we simply
|
||
close the database handle, sleep for a short period of time
|
||
and then continue processing. This gives the Replication Manager a chance to
|
||
create the database so that our replica can continue
|
||
operations.
|
||
</p>
|
||
<pre class="programlisting"> try {
|
||
db = dbenv.openDatabase
|
||
(null, RepConfig.progname, null, dbconf);
|
||
} catch (java.io.FileNotFoundException e) {
|
||
<strong class="userinput"><code>System.err.println("no stock database available yet.");
|
||
if (db != null) {
|
||
db.close(true);
|
||
db = null;
|
||
}
|
||
try {
|
||
Thread.sleep(RepConfig.SLEEPTIME);
|
||
} catch (InterruptedException ie) {}
|
||
continue;</code></strong>
|
||
}
|
||
} </pre>
|
||
<p>
|
||
Next we modify our prompt, so that if the local process is running
|
||
as a replica, we can tell from the shell that the prompt is for a
|
||
read-only process.
|
||
</p>
|
||
<pre class="programlisting"> BufferedReader stdin =
|
||
new BufferedReader(new InputStreamReader(System.in));
|
||
|
||
// listen for input, and add it to the database.
|
||
System.out.print("QUOTESERVER");
|
||
<strong class="userinput"><code>if (!dbenv.getIsMaster())
|
||
System.out.print("(read-only)");
|
||
System.out.print("> ");</code></strong>
|
||
System.out.flush();
|
||
String nextline = null;
|
||
try {
|
||
nextline = stdin.readLine();
|
||
} catch (IOException ioe) {
|
||
System.err.println("Unable to get data from stdin");
|
||
break;
|
||
}
|
||
String[] words = nextline.split("\\s"); </pre>
|
||
<p>
|
||
When we collect data from the prompt, there is a case that says
|
||
if no data is entered then show the entire stocks database.
|
||
This display is performed by our
|
||
<code class="function">print_stocks()</code>
|
||
|
||
<span>method</span>
|
||
(which has not
|
||
required a modification since we first introduced it in
|
||
<a class="xref" href="simpleprogramlisting.html#printstocks_c" title="Method: SimpleTxn.printStocks()">
|
||
|
||
|
||
<span>Method: SimpleTxn.printStocks()</span>
|
||
</a>).
|
||
</p>
|
||
<p>
|
||
When we call
|
||
|
||
<span><code class="function">printStocks()</code>, </span>
|
||
we check for a dead replication handle. Dead
|
||
replication handles happen whenever a replication election
|
||
results in a previously committed transaction becoming
|
||
invalid. This is an error scenario caused by a new master having a
|
||
slightly older version of the data than the original
|
||
master and so all replicas must modify their database(s) to
|
||
reflect that of the new master. In this situation, some
|
||
number of previously committed transactions may have to be
|
||
unrolled. From the replica's perspective, the database
|
||
handles should all be closed and then opened again.
|
||
</p>
|
||
<pre class="programlisting"> // A blank line causes the DB to be dumped to stdout.
|
||
if (words.length == 0 ||
|
||
(words.length == 1 && words[0].length() == 0)) {
|
||
try {
|
||
printStocks(db);
|
||
<strong class="userinput"><code>} catch (DeadlockException de) {
|
||
continue;
|
||
// Dead replication handles are caused by an election
|
||
// resulting in a previously committing read becoming
|
||
// invalid. Close the db handle and reopen.
|
||
} catch (ReplicationHandleDeadException rhde) {
|
||
db.close(true); // close no sync.
|
||
db = null;
|
||
continue;</code></strong>
|
||
} catch (DatabaseException e) {
|
||
System.err.println("Got db exception reading " +
|
||
"replication DB: " + e.toString());
|
||
break;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (words.length == 1 &&
|
||
(words[0].compareToIgnoreCase("quit") == 0 ||
|
||
words[0].compareToIgnoreCase("exit") == 0)) {
|
||
break;
|
||
} else if (words.length != 2) {
|
||
System.err.println("Format: TICKER VALUE");
|
||
continue;
|
||
} </pre>
|
||
<p>
|
||
That done, we need to add a little error checking to our
|
||
command prompt to make sure the user is not attempting to
|
||
modify the database at a replica. Remember, replicas must never
|
||
modify their local databases on their own. This guards against
|
||
that happening due to user input at the prompt.
|
||
</p>
|
||
<pre class="programlisting"> <strong class="userinput"><code>if (!dbenv.getIsMaster()) {
|
||
System.err.println("Can't update client.");
|
||
continue;
|
||
}</code></strong>
|
||
|
||
DatabaseEntry key = new DatabaseEntry(words[0].getBytes());
|
||
DatabaseEntry data = new DatabaseEntry(words[1].getBytes());
|
||
|
||
db.put(null, key, data);
|
||
}
|
||
if (db != null)
|
||
db.close(true);
|
||
return 0;
|
||
} </pre>
|
||
<p>
|
||
With that completed, we are all done updating our application
|
||
for replication.
|
||
|
||
The only remaining
|
||
|
||
|
||
<span>method, <code class="function">printStocks()</code>,</span>
|
||
|
||
is unmodified from when we
|
||
originally introduced it. For details on that function, see
|
||
<a class="xref" href="simpleprogramlisting.html#printstocks_c" title="Method: SimpleTxn.printStocks()">
|
||
|
||
|
||
<span>Method: SimpleTxn.printStocks()</span>
|
||
</a>.
|
||
</p>
|
||
<div class="sect2" lang="en" xml:lang="en">
|
||
<div class="titlepage">
|
||
<div>
|
||
<div>
|
||
<h3 class="title"><a id="runningit"></a>Running It</h3>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>
|
||
To run our replicated application, we need to make
|
||
sure each participating environment has its own unique
|
||
home directory. We can do this by running
|
||
each site on a separate networked machine, but that
|
||
is not strictly necessary; multiple instances of this
|
||
code can run on the same machine provided the
|
||
environment home restriction is observed.
|
||
</p>
|
||
<p>
|
||
To run a process, make sure the environment home exists and
|
||
then start the process using the <code class="literal">-h</code>
|
||
option to specify that directory. You must also use the
|
||
<code class="literal">-l</code> or <code class="literal">-L</code> option to
|
||
identify the local host and port that this process will use
|
||
to listen for replication messages (-L means that this is a
|
||
group creator), and the <code class="literal">-r</code> option to
|
||
identify the other processes in the replication group.
|
||
Finally, use the <code class="literal">-p</code> option to specify a
|
||
priority. The process that you designate to have the
|
||
highest priority will become the master.
|
||
</p>
|
||
<pre class="programlisting">> mkdir env1
|
||
> java db.repquote_gsg.RepQuoteExampleGSG -h env1 -L localhost:8080 \
|
||
-p 10
|
||
No stock database yet available.
|
||
No stock database yet available. </pre>
|
||
<p>
|
||
Now, start another process. This time, change the environment
|
||
home to something else, use the <code class="literal">-l</code> flag to at
|
||
least change the port number the process is listening on, and
|
||
use the <code class="literal">-r</code> option to identify the host and
|
||
port of the other replication process:
|
||
</p>
|
||
<pre class="programlisting">> mkdir env2
|
||
> java db.repquote_gsg.RepQuoteExampleGSG -h env2 -l localhost:8081 \
|
||
-r localhost:8080 -p 20</pre>
|
||
<p>
|
||
After a short pause, the second process should display the master
|
||
prompt:
|
||
</p>
|
||
<pre class="programlisting">
|
||
QUOTESERVER > </pre>
|
||
<p>
|
||
And the first process should
|
||
display the read-only prompt:
|
||
</p>
|
||
<pre class="programlisting">
|
||
QUOTESERVER (read-only)> </pre>
|
||
<p>
|
||
Now go to the master process and give it a couple of stocks and stock
|
||
prices:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER> FAKECO 9.87
|
||
QUOTESERVER> NOINC .23
|
||
QUOTESERVER> </pre>
|
||
<p>
|
||
Then, go to the replica and hit <strong class="userinput"><code>return</code></strong> at the prompt to
|
||
see the new values:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER (read-only)>
|
||
Symbol Price
|
||
====== =====
|
||
FAKECO 9.87
|
||
NOINC .23
|
||
QUOTESERVER (read-only)> </pre>
|
||
<p>
|
||
Doing the same at the master results in the same thing:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER>
|
||
Symbol Price
|
||
====== =====
|
||
FAKECO 9.87
|
||
NOINC .23
|
||
QUOTESERVER> </pre>
|
||
<p>
|
||
You can change a stock by simply entering the stock value and
|
||
new price at the master's prompt:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER> FAKECO 10.01
|
||
QUOTESERVER> </pre>
|
||
<p>
|
||
Then, go to either the master or the replica to see the updated
|
||
database. On the master:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER>
|
||
Symbol Price
|
||
====== =====
|
||
FAKECO 10.01
|
||
NOINC .23
|
||
QUOTESERVER> </pre>
|
||
<p>
|
||
And on the replica:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER (read-only)>
|
||
Symbol Price
|
||
====== =====
|
||
FAKECO 10.01
|
||
NOINC .23
|
||
QUOTESERVER (read-only)> </pre>
|
||
<p>
|
||
Finally, to quit the applications, simply type
|
||
<code class="literal">quit</code> at both prompts. On the replica:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER (read-only)> quit
|
||
> </pre>
|
||
<p>
|
||
And on the master as well:
|
||
</p>
|
||
<pre class="programlisting">QUOTESERVER> quit
|
||
> </pre>
|
||
</div>
|
||
</div>
|
||
<div class="navfooter">
|
||
<hr />
|
||
<table width="100%" summary="Navigation footer">
|
||
<tr>
|
||
<td width="40%" align="left"><a accesskey="p" href="processingloop.html">Prev</a> </td>
|
||
<td width="20%" align="center">
|
||
<a accesskey="u" href="fwrkmasterreplica.html">Up</a>
|
||
</td>
|
||
<td width="40%" align="right"> <a accesskey="n" href="addfeatures.html">Next</a></td>
|
||
</tr>
|
||
<tr>
|
||
<td width="40%" align="left" valign="top">Processing Loop </td>
|
||
<td width="20%" align="center">
|
||
<a accesskey="h" href="index.html">Home</a>
|
||
</td>
|
||
<td width="40%" align="right" valign="top"> Chapter 5. Additional Features</td>
|
||
</tr>
|
||
</table>
|
||
</div>
|
||
</body>
|
||
</html>
|