Log truncation. (But no checkpoints, so it doesn't get called... it does pass testing though. :)

lladd/constants.h

@@ -59,6 +59,7 @@ terms specified in this license.
 /*#define DEBUG 1*/
 
 #define LOG_FILE "logfile.txt"
+#define LOG_FILE_SCRATCH "logfile.txt~"
 #define STORE_FILE "storefile.txt"
 #define BLOB0_FILE "blob0_file.txt"
 #define BLOB1_FILE "blob1_file.txt"

lladd/logger.h

@@ -1,126 +0,0 @@
-/*---
-This software is copyrighted by the Regents of the University of
-California, and other parties. The following terms apply to all files
-associated with the software unless explicitly disclaimed in
-individual files.
-
-The authors hereby grant permission to use, copy, modify, distribute,
-and license this software and its documentation for any purpose,
-provided that existing copyright notices are retained in all copies
-and that this notice is included verbatim in any distributions. No
-written agreement, license, or royalty fee is required for any of the
-authorized uses. Modifications to this software may be copyrighted by
-their authors and need not follow the licensing terms described here,
-provided that the new terms are clearly indicated on the first page of
-each file where they apply.
-
-IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
-FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
-ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
-DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
-INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
-NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
-THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
-MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
-
-GOVERNMENT USE: If you are acquiring this software on behalf of the
-U.S. government, the Government shall have only "Restricted Rights" in
-the software and related documentation as defined in the Federal
-Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
-acquiring the software on behalf of the Department of Defense, the
-software shall be classified as "Commercial Computer Software" and the
-Government shall have only "Restricted Rights" as defined in Clause
-252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
-authors grant the U.S. Government and others acting in its behalf
-permission to use and distribute the software in accordance with the
-terms specified in this license.
----*/
-
-/**
- * @file
- *
- * External interaface to Logging API.  (Only exposes operations used
- * to add entries to the log.)
- *
- * Logger is the front end for logging actions -- whatever must be done (ie
- * write to the log tail; flushing for commits) is handled in these functions
- *
- * @deprecated 
- * @see logger2.h
- *
- * @ingroup LLADD_CORE
- * $Id$
- * 
- * *************/
-
-
-#ifndef __LOGGER_H__
-#define __LOGGER_H__
-
-#include "transactional.h"
-#include "operations.h"
-
-/**
-  Does NOT write a transaction begin; rather, just returns that the
-  LSN of a potential entry is -1 so the next command will have a
-  prevLSN of -1.  (Althoug this is currently a no-op, it's possible
-  that some other logging scheme would actually write begin records.)
-*/
-long LogTransBegin(Transaction t);
-
-/**
- * logs the fact that a rid has been allocated for a transaction
- */
-long LogTransAlloc(long prevLSN, int xid, recordid rid);
-
-/**
-  Write a transaction COMMIT to the log tail, then flush the log tail immediately to disk
-
-  @return the LSN of this entry
-*/
-long LogTransCommit(long prevLSN, int xid);
-
-/**
-  Write a transaction ABORTto the log tail
-
-  @return returns the LSN of this entry
-*/
-long LogTransAbort(long prevLSN, int xid);
-
-/**
-  LogUpdate writes an UPDATE log record to the log tail
-
-  returns the LSN of this entry
-*/
-long LogUpdate (long prevLSN, int xid, recordid rid, Operation op, const void *args);
-
-/**
-   Write a compensation log record.  These records are used to allow
-   for efficient recovery, and possibly for log truncation.  They
-   record the completion of undo operations.
-*/
-long LogCLR (long prevLSN, int xid, long ulLSN, recordid ulRID, long ulPrevLSN);
-
-/**
-   Write a end transaction record @ todo What does this do exactly?  Indicate completion of aborts?
-*/
-long LogEnd (long prevLSN, int xid);
-
-/*
-  Starts a new log stream, possibly other stuff can go here too?
-*/
-void logInit();
-
-/*
-  Called when ALL transactions are guaranteed to be completed (either
-  committed or aborted) and no new ones can be had. So therefore we
-  can close the log streamer and delete the log file.  @todo Doesn't
-  delete logs right now.  (For debugging)
-*/
-void logDeinit();
-
-#endif

lladd/logger/logWriter.h

@@ -108,6 +108,41 @@ void syncLog();
 */
 lsn_t flushedLSN();
 
+/**
+   Truncates the log file.  In the single-threaded case, this works as
+   follows:
+
+   First, the LSN passed to this function, minus sizeof(lsn_t) is
+   written to a new file, called logfile.txt~.  (If logfile.txt~
+   already exists, then it is truncated.)
+
+   Next, the contents of the log, starting with the LSN passed into
+   this function are copied to logfile.txt~
+
+   Finally, logfile.txt~ is moved on top of logfile.txt
+
+   As long as the move system call is atomic, this function should
+   maintain the system's durability.
+
+   The multithreaded case is a bit more complicated, as we need
+   to deal with latching:
+
+   With no lock, copy the log.  Upon completion, if the log has grown,
+   then copy the part that remains.  Next, obtain a read/write latch
+   on the logfile, and copy any remaining portions of the log.
+   Perform the move, and release the latch.
+
+*/
+
+int truncateLog(lsn_t);
+
+
+/** 
+    @return The LSN of the first entry in the log file.  (If the file
+    is empty, this returns the LSN of the log entry that would be
+    created if writeLogEntry were called.)
+*/
+lsn_t firstLogEntry();
 /**
   Close the log stream
 */

src/lladd/logger/logHandle.c

@@ -54,7 +54,10 @@ static void set_offsets(LogHandle * h, LogEntry * e, lsn_t lastRead);
 /*-------------------------------------------------------*/
 
 LogHandle getLogHandle() {
-  return getGuardedHandle(sizeof(lsn_t), NULL, NULL);
+
+  lsn_t lsn = firstLogEntry();
+
+  return getGuardedHandle(lsn, NULL, NULL);
 }
 
 LogHandle getLSNHandle(lsn_t lsn) {

src/lladd/logger/logWriter.c

@@ -43,7 +43,7 @@ terms specified in this license.
 #include <lladd/logger/logHandle.h>
 #include <assert.h>
 #include <config.h>
-
+#include <stdlib.h>
 #include <malloc.h>
 #include <unistd.h>
 /** 
@@ -71,10 +71,14 @@ static lsn_t nextAvailableLSN = 0;
 static int writeLogEntryIsReady = 0;
 static lsn_t maxLSNEncountered = sizeof(lsn_t);
 
+static lsn_t global_offset;
 
 
+/** 
+    @todo Put myFseek, myFwrite in their own file, and make a header for it... */
 
-
+void myFwrite(const void * dat, size_t size, FILE * f);
+long myFseek(FILE * f, long offset, int whence);
 int openLogWriter() {
   log = fopen(LOG_FILE, "a+");
   if (log==NULL) {
@@ -110,6 +114,12 @@ int openLogWriter() {
       assert(0);
       return FILE_WRITE_OPEN_ERROR;
     }
+    global_offset = 0;
+  } else {
+    int count;
+    myFseek(log, 0, SEEK_SET);
+    count = fread(&global_offset, sizeof(lsn_t), 1, log);
+    assert(count == 1);
   }
 
   return 0;
@@ -144,11 +154,18 @@ int writeLogEntry(LogEntry * e) {
   }
   
   if(!writeLogEntryIsReady) {
-    LogHandle lh = getLSNHandle(maxLSNEncountered);
+    LogHandle lh;
     LogEntry * le;
     
+    assert(maxLSNEncountered >= sizeof(lsn_t));
+
+    lh = getLSNHandle(maxLSNEncountered);
+
+    nextAvailableLSN = maxLSNEncountered;
+
     while((le = nextInLog(&lh))) {
-      nextAvailableLSN = le->LSN + sizeofLogEntry(le) + sizeof(lsn_t);
+      nextAvailableLSN = le->LSN + sizeofLogEntry(le) + sizeof(size_t);;
+      free(le);
     }
     writeLogEntryIsReady = 1;
   }
@@ -156,8 +173,20 @@ int writeLogEntry(LogEntry * e) {
 
 
   /* Set the log entry's LSN. */
+
+#ifdef DEBUGGING
   fseek(log, 0, SEEK_END);
   e->LSN = ftell(log);
+  if(nextAvailableLSN != e->LSN) {
+    assert(nextAvailableLSN <= e->LSN);
+    DEBUG("Detected log truncation:  nextAvailableLSN = %ld, but log length is %ld.\n", (long)nextAvailableLSN, e->LSN);
+  }
+#endif
+
+  e->LSN = nextAvailableLSN;
+  fseek(log, nextAvailableLSN - global_offset, SEEK_SET); 
+      
+  nextAvailableLSN += (size + sizeof(size_t));
 
   /* Print out the size of this log entry.  (not including this item.) */
   nmemb = fwrite(&size, sizeof(size_t), 1, log);
@@ -272,8 +301,93 @@ LogEntry * readLSNEntry(lsn_t LSN) {
     }
   }
 
-  fseek(log, LSN, SEEK_SET);
+  fseek(log, LSN - global_offset, SEEK_SET);
   ret = readLogEntry();
 
   return ret;
 }
+
+int truncateLog(lsn_t LSN) {
+  FILE *tmpLog = fopen(LOG_FILE_SCRATCH, "w+");  /* w+ = truncate, and open for writing. */
+
+  LogEntry * le;
+  LogHandle lh;
+
+  long size;
+
+  int count;
+
+
+  if (tmpLog==NULL) {
+    assert(0);
+    /*there was an error opening this file */
+    perror("logTruncate() couldn't create scratch log file!");
+    return FILE_WRITE_OPEN_ERROR;
+  }
+
+  /* Need to write LSN - sizeof(lsn_t) to make room for the offset in
+     the file.  If we truncate to lsn 10, we'll put lsn 10 in position
+     4, so the file offset is 6. */
+  LSN -= sizeof(lsn_t);  
+
+  DEBUG("Truncate(%ld) new file offset = %ld\n", LSN + sizeof(lsn_t), LSN);
+
+  myFwrite(&LSN, sizeof(lsn_t), tmpLog);
+  
+  LSN += sizeof(lsn_t);
+  
+  lh = getLSNHandle(LSN);
+  
+  while((le = nextInLog(&lh))) {
+    size = sizeofLogEntry(le);
+    myFwrite(&size, sizeof(lsn_t), tmpLog);
+    myFwrite(le, size, tmpLog);
+    free (le);
+  } 
+  
+  fflush(tmpLog);
+#ifdef HAVE_FDATASYNC
+  fdatasync(fileno(tmpLog));
+#else
+  fsync(fileno(tmpLog));
+#endif
+
+  fclose(log);  /* closeLogWriter calls sync, but we don't need to. :) */
+  fclose(tmpLog); 
+ 
+  if(rename(LOG_FILE_SCRATCH, LOG_FILE)) {
+    perror("Log truncation failed!");
+    abort();
+  }
+
+
+  log = fopen(LOG_FILE, "a+");
+  if (log==NULL) {
+    abort();
+    /*there was an error opening this file */
+    return FILE_WRITE_OPEN_ERROR;
+  }
+
+  myFseek(log, 0, SEEK_SET);
+  count = fread(&global_offset, sizeof(lsn_t), 1, log);
+  assert(count == 1);
+
+
+  return 0;
+
+}
+
+lsn_t firstLogEntry() {
+  return global_offset + sizeof(lsn_t);
+}
+
+void myFwrite(const void * dat, size_t size, FILE * f) {
+  int nmemb = fwrite(dat, size, 1, f);
+  /* test */
+  if(nmemb != 1) {
+    perror("myFwrite");
+    abort();
+    /*    return FILE_WRITE_OPEN_ERROR; */
+  }
+  
+}

test/lladd/check_logWriter.c

@@ -47,6 +47,8 @@ terms specified in this license.
 #include <lladd/logger/logHandle.h>
 #include <lladd/transactional.h>
 
+#include "../check_includes.h"
+
 #define LOG_NAME   "check_logWriter.log"
 
 static void setup_log() {
@@ -167,6 +169,66 @@ START_TEST(logHandleColdReverseIterator) {
 }
 END_TEST
 
+/** 
+    @test
+
+    Build a simple log, truncate it, and then test the logWriter routines against it.
+*/
+START_TEST(logWriterTruncate) {
+  LogEntry * le;
+  LogEntry * le2;
+  LogEntry * le3 = NULL;
+  LogEntry * tmp;
+
+  LogHandle lh = getLogHandle();
+  int i = 0;
+  setup_log();
+
+  while(i < 234) {
+    i++;
+    le = nextInLog(&lh);
+  }
+ 
+  le2 = nextInLog(&lh);
+  i = 0;
+  while(i < 23) {
+    i++;
+    le3 = nextInLog(&lh);
+  }
+  
+
+  truncateLog(le->LSN);
+  
+  tmp = readLSNEntry(le->LSN);
+
+  fail_unless(NULL != tmp, NULL);
+  fail_unless(tmp->LSN == le->LSN, NULL);
+  
+  tmp = readLSNEntry(le2->LSN);
+
+  fail_unless(NULL != tmp, NULL);
+  fail_unless(tmp->LSN == le2->LSN, NULL);
+
+  tmp = readLSNEntry(le3->LSN);
+
+  fail_unless(NULL != tmp, NULL);
+  fail_unless(tmp->LSN == le3->LSN, NULL);
+
+
+  lh = getLogHandle();
+  
+  i = 0;
+
+  while((le = nextInLog(&lh))) {
+    i++;
+  }
+
+
+  fail_unless(i == (3000 - 234 + 1), NULL);
+  
+
+} END_TEST
+
 Suite * check_suite(void) {
   Suite *s = suite_create("logWriter");
   /* Begin a new test */
@@ -176,9 +238,13 @@ Suite * check_suite(void) {
 
   tcase_add_test(tc, logWriterTest);
   tcase_add_test(tc, logHandleColdReverseIterator);
+  tcase_add_test(tc, logWriterTruncate);
 
   /* --------------------------------------------- */
   
+  tcase_add_checked_fixture(tc, setup, teardown);
+
+
   suite_add_tcase(s, tc);
   return s;
 }