greg/stasis-aries-wal
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implemented indirect pages, and fixed some bugs here and there. (Still need to add transactional support for indirect pages...)

Browse source
This commit is contained in:
Sears Russell 2004-07-31 00:27:55 +00:00
parent a74c499dd7
commit e51759b517
10 changed files with 379 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lladd/transactional.h
View file

@ -95,6 +95,15 @@ typedef struct {
long size;
} recordid;
/**
If a recordid's slot field is set to this, then the recordid
represents an array of fixed-length records starting at slot zero
of the recordid's page.
@todo Support read-only arrays of variable length records, and then
someday read / write / insert / delete arrays...
*/
#define RECORD_ARRAY (-1)
#include "operations.h"

2
src/lladd/Makefile.am
View file

@ -3,6 +3,6 @@
lib_LIBRARIES=liblladd.a
#liblladd_a_LIBADD=logger/liblogger.a operations/liboperations.a
# removed: recovery.c transactional.c logger.c logger/logparser.c logger/logstreamer.c
liblladd_a_SOURCES=common.c stats.c io.c bufferManager.c linkedlist.c operations.c pageFile.c pageCache.c page.c blobManager.c recovery2.c transactional2.c logger/logEntry.c logger/logWriter.c logger/logHandle.c logger/logger2.c operations/decrement.c operations/increment.c operations/prepare.c operations/set.c operations/alloc.c page/slotted.c #operations/lladdhash.c
liblladd_a_SOURCES=common.c stats.c io.c bufferManager.c linkedlist.c operations.c pageFile.c pageCache.c page.c blobManager.c recovery2.c transactional2.c logger/logEntry.c logger/logWriter.c logger/logHandle.c logger/logger2.c operations/decrement.c operations/increment.c operations/prepare.c operations/set.c operations/alloc.c page/slotted.c page/indirect.c #operations/lladdhash.c
AM_CFLAGS= -g -Wall -pedantic -std=gnu99

65
src/lladd/page.c
View file

@ -40,6 +40,34 @@ permission to use and distribute the software in accordance with the
terms specified in this license.
---*/
/**
@file
Generic page interface. This file handles updates to the LSN, but
leaves finer grained concurrency to the implementor of each of the
page types. This interface's primary purpose is to wrap common
functionality together, and to delegate responsibility for page
handling to other modules.
Latching summary:
Each page has an associated read/write lock. This lock only
protects the internal layout of the page, and the members of the
page struct. Here is how it is held in various circumstances:
Record allocation: Write lock
Record read: Read lock
Read LSN Read lock
Record write *READ LOCK*
Write LSN Write lock
Any circumstance where these locks are held during an I/O operation
is a bug.
*/
/* _XOPEN_SOURCE is needed for posix_memalign */
#define _XOPEN_SOURCE 600
#include <stdlib.h>
@ -72,8 +100,8 @@ static int nextPage = 0;
only place where pageRalloc is called, pageRalloc does not obtain
this lock.
*/
static pthread_mutex_t lastFreepage_mutex;
static unsigned int lastFreepage = 0;
pthread_mutex_t lastFreepage_mutex;
unsigned int lastFreepage = 0;
@ -90,10 +118,13 @@ Page pool[MAX_BUFFER_SIZE+1];
*
* @param page You must have a writelock on page before calling this function.
*/
void pageWriteLSN(Page * page) {
void pageWriteLSN(Page * page, lsn_t lsn) {
/* unlocked since we're only called by a function that holds the writelock. */
/* *(long *)(page->memAddr + START_OF_LSN) = page->LSN; */
if(page->LSN < lsn) {
page->LSN = lsn;
*lsn_ptr(page) = page->LSN;
}
}
/**
@ -191,7 +222,18 @@ void pageAbort(int xid) {
}
/** @todo ralloc ignores it's xid parameter; change the interface? */
int pageAllocMultiple(int newPageCount) {
pthread_mutex_lock(&lastFreepage_mutex);
int ret = lastFreepage+1;
lastFreepage += newPageCount;
pthread_mutex_unlock(&lastFreepage_mutex);
return ret;
}
/** @todo ralloc ignores it's xid parameter; change the interface?
@todo ralloc doesn't set the page type, and interacts poorly with other methods that allocate pages.
*/
recordid ralloc(int xid, long size) {
recordid ret;
@ -199,13 +241,16 @@ recordid ralloc(int xid, long size) {
/* DEBUG("Rallocing record of size %ld\n", (long int)size); */
assert(size < BLOB_THRESHOLD_SIZE || size == BLOB_SLOT);
assert(size < BLOB_THRESHOLD_SIZE);
pthread_mutex_lock(&lastFreepage_mutex);
while(freespace(p = loadPage(lastFreepage)) < size ) {
p = loadPage(lastFreepage);
*page_type_ptr(p) = SLOTTED_PAGE;
while(freespace(p) < size ) {
releasePage(p);
lastFreepage++;
p = loadPage(lastFreepage);
*page_type_ptr(p) = SLOTTED_PAGE;
}
ret = pageRalloc(p, size);
@ -272,10 +317,8 @@ void writeRecord(int xid, Page * p, lsn_t lsn, recordid rid, const void *dat) {
writelock(p->rwlatch, 225); /* Need a writelock so that we can update the lsn. */
if(p->LSN < lsn) {
p->LSN = lsn;
pageWriteLSN(p);
}
pageWriteLSN(p, lsn);
unlock(p->rwlatch);
}

66
src/lladd/page.h
View file

@ -55,30 +55,28 @@ terms specified in this license.
* structure should be seperated, and each page should have a 'type'
* slot so that we can implement multiple page types on top of LLADD.
Slotted page layout:
END:
lsn (4 bytes)
type (2 bytes)
free space (2 bytes)
num of slots (2 bytes)
freelist head(2 bytes)
slot 0 (2 bytes)
slot 1 (2 bytes)
...
slot n (2 bytes)
...
unused
...
record n (x bytes)
...
record 0 (y bytes)
record 1 (z bytes)
START
**/
STRUCTURE OF A GENERIC PAGE
<pre>
+----------------------------------------------------------------------+
| |
| USABLE SPACE |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| +-----------+-----+
| | page type | LSN |
+----------------------------------------------------+-----------+-----+
</pre>
*/
#ifndef __PAGE_H__
#define __PAGE_H__
@ -94,12 +92,10 @@ Slotted page layout:
BEGIN_C_DECLS
/*
#define LSN_SIZE sizeof(lsn_t)
#define START_OF_LSN (PAGE_SIZE - LSN_SIZE)
#define PAGE_TYPE_SIZE 0
#define START_OF_PAGE_TYPE (START_OF_LSN - PAGE_TYPE_SIZE)
#define USABLE_SPACE_SIZE (START_OF_PAGE_TYPE)*/
#define UNINITIALIZED_PAGE 0
#define SLOTTED_PAGE 1
#define INDIRECT_PAGE 2
#define lsn_ptr(page) (((lsn_t *)(&((page)->memAddr[PAGE_SIZE])))-1)
#define page_type_ptr(page) (((int*)lsn_ptr((page)))-1)
@ -107,8 +103,9 @@ BEGIN_C_DECLS
#define shorts_from_end(page, count) (((short*)end_of_usable_space_ptr((page)))-(count))
#define bytes_from_start(page, count) (((byte*)((page)->memAddr))+(count))
#define ints_from_start(page, count) (((int*)((page)->memAddr))+(count))
#define USABLE_SIZE_OF_PAGE (PAGE_SIZE - sizeof(lsn_t) - sizeof(int))
/*#define invalidateSlot(page, n) (*slot_ptr((page), (n)) = INVALID_SLOT)*/
@ -207,7 +204,7 @@ void pageDeInit();
* as a parameter a Page. The Page struct contains the new LSN and the page
* number to which the new LSN must be written to.
*/
/*void pageWriteLSN(Page page);*/
void pageWriteLSN(Page * page, lsn_t lsn);
/**
* assumes that the page is already loaded in memory. It takes
@ -279,6 +276,11 @@ void pageAbort(int xid);
Page* pageAlloc(int id);
void pageRealloc(Page * p, int id);
/** Allocates a set of contiguous pages on disk. Has nothing to do with pageAlloc.
@todo need a better naming convention for pageAlloc (alloc's memory) and pageAllocMultiple (alloc's disk)
*/
int pageAllocMultiple(int newPageCount) ;
int pageGetSlotType(Page * p, int slot, int type);
void pageSetSlotType(Page * p, int slot, int type);

161
src/lladd/page/indirect.c Normal file
View file

@ -0,0 +1,161 @@
#include "indirect.h"
#include "slotted.h"
#include <math.h>
#include <string.h>
#include <assert.h>
#include "../blobManager.h"
#include "../page.h"
void indirectInitialize(Page * p, int height) {
*level_ptr(p) = height;
*page_type_ptr(p) = INDIRECT_PAGE;
memset(p->memAddr, INVALID_SLOT, ((int)level_ptr(p)) - ((int)p->memAddr));
}
recordid dereferenceRID(recordid rid) {
Page * this = loadPage(rid.page);
int offset = 0;
int max_slot;
while(*page_type_ptr(this) == INDIRECT_PAGE) {
int i = 0;
for(max_slot = *maxslot_ptr(this, i); ( max_slot + offset ) <= rid.slot; max_slot = *maxslot_ptr(this, i)) {
i++;
assert(max_slot != INVALID_SLOT);
}
if(i) {
offset += *maxslot_ptr(this, i - 1);
} /** else, the adjustment to the offset is zero */
int nextPage = *page_ptr(this, i);
releasePage(this);
this = loadPage(nextPage);
}
rid.page = this->id;
rid.slot -= offset;
releasePage(this);
return rid;
}
#define min(x, y) ((x) < (y) ? (x) : (y))
/** Would be static, but there is a unit test for this function */
unsigned int calculate_level (unsigned int number_of_pages) {
long long tmp = INDIRECT_POINTERS_PER_PAGE;
unsigned int level = 1;
while(tmp < number_of_pages) {
tmp *= INDIRECT_POINTERS_PER_PAGE;
level++;
}
return level;
}
recordid rallocMany(int parentPage, lsn_t lsn, int recordSize, int recordCount) {
/* How many levels of pages do we need? */
int physical_size;
recordid rid;
if(recordSize > BLOB_THRESHOLD_SIZE) {
physical_size = sizeof(blob_record_t);
} else {
physical_size = recordSize;
}
int records_per_page = (USABLE_SIZE_OF_PAGE - SLOTTED_PAGE_HEADER_OVERHEAD)
/ (physical_size + SLOTTED_PAGE_OVERHEAD_PER_RECORD); /* we need to take the floor */
int number_of_pages = (int)ceil( (double)recordCount / (double)records_per_page); /* need to take ceiling here */
if(number_of_pages > 1) {
int level = calculate_level(number_of_pages);
DEBUG("recordsize = %d, physicalsize = %d, recordCount = %d, level = %d\n",
recordSize, physical_size, recordCount, level);
/* OK, now allocate the pages. */
int next_level_records_per_page = records_per_page;
for(int i = 0; i < (level - 1); i++) {
next_level_records_per_page *= INDIRECT_POINTERS_PER_PAGE;
}
int newPageCount = (int)ceil((double)recordCount / (double)next_level_records_per_page);
int firstChildPage = pageAllocMultiple(newPageCount);
int tmpRecordCount = recordCount;
int thisChildPage = firstChildPage;
while(tmpRecordCount > 0) {
rallocMany(thisChildPage, lsn, recordSize, min(tmpRecordCount, next_level_records_per_page));
tmpRecordCount -= next_level_records_per_page;
thisChildPage ++;
}
assert((thisChildPage-firstChildPage)== newPageCount);
tmpRecordCount = recordCount;
Page * p = loadPage(parentPage);
writelock(p->rwlatch, 99);
indirectInitialize(p, level);
int i = 0;
for(tmpRecordCount = recordCount; tmpRecordCount > 0; tmpRecordCount -= next_level_records_per_page) {
*page_ptr(p, i) = firstChildPage + i;
if(i) {
*maxslot_ptr(p, i) = *maxslot_ptr(p, i-1) + min(tmpRecordCount, next_level_records_per_page);
} else {
*maxslot_ptr(p, i) = min(tmpRecordCount, next_level_records_per_page);
}
i++;
}
assert(i == newPageCount);
pageWriteLSN(p, lsn);
unlock(p->rwlatch);
releasePage(p);
rid.page = parentPage;
rid.slot = RECORD_ARRAY;
rid.size = recordSize;
} else {
DEBUG("recordsize = %d, recordCount = %d, level = 0 (don't need indirect pages)\n", recordSize, recordCount);
Page * p = loadPage(parentPage);
writelock(p->rwlatch, 127);
pageInitialize(p);
unlock(p->rwlatch);
for(int i = 0; i < recordCount; i++) {
pageRalloc(p, recordSize);
}
writelock(p->rwlatch, 127);
pageWriteLSN(p, lsn);
unlock(p->rwlatch);
releasePage(p);
rid.page = parentPage;
rid.slot = RECORD_ARRAY;
rid.size = recordSize;
}
return rid;
}

52
src/lladd/page/indirect.h Normal file
View file

@ -0,0 +1,52 @@
/**
@file Indirect block implementation.
On disk layout of indirect blocks:
END
lsn (2 bytes)
type = 2 (2 bytes)
level (2 bytes)
...
block1 = {pageid, maxslot} (8 bytes)
block0 = {pageid, maxslot} (8 bytes)
START
If blockN has pageid = INVALID_SLOT, then block(N-1) is the last
indirect block that has been allocated.
maxslot is the first slot number that would not fit on this page. (If the slot exists, then it must be on the next page).
The 'level' field indicates how many levels of indirect blocks lie
below this block. level = 1 means that the pageid's point to 'normal'
pages. (They may be slotted (type = 1), or provided by some other
implementation).
*/
#include <lladd/common.h>
#include "../page.h"
#ifndef __LLADD_PAGE_INDIRECT_H
#define __LLADD_PAGE_INDIRECT_H
BEGIN_C_DECLS
#define level_ptr(page) shorts_from_end((page), 3)
#define page_ptr(page, offset) ints_from_start((page), 2*(offset))
#define maxslot_ptr(page, offset) ints_from_start((page), 2*(offset)+1)
#define INDIRECT_POINTERS_PER_PAGE (USABLE_SIZE_OF_PAGE / 16)
/**
Translates a recordid that points to an indirect block into the
physical location of the record.
*/
recordid dereferenceRID(recordid rid);
void indirectInitialize(Page * p, int height);
recordid rallocMany(int parentPage, lsn_t lsn, int recordSize, int recordCount);
END_C_DECLS
#endif /*__LLADD_PAGE_INDIRECT_H*/

46
src/lladd/page/slotted.c
View file

@ -1,49 +1,6 @@
/************************************************************************
* implementation of pages
/** $Id$ */
STRUCTURE OF A PAGE
+-------------------------------------------+-----------------------+--+
| DATA SECTION +--------->| RID: (PAGE, 0) | |
| +-----------------+ | +-----------------------+ |
| +-->| RID: (PAGE, 1) | | |
| | +-----------------+ | |
| | | |
| +-----------------+ | +----------------------------+
| | | +--->| RID: (PAGE, n) |
| | | | +----------------------------+
|======================================================================|
|^ FREE SPACE | | | |
|+-----------------------|-------|---|--------------------+ |
| | | | | |
| +-------------|-------|---+ | |
| | | | | |
| +---|---+-----+---|---+---|---+--------------+-----|------+-----+
| | slotn | ... | slot1 | slot0 | num of slots | free space | LSN |
+------+-------+-----+-------+-------+--------------+------------+-----+
NOTE:
- slots are zero indexed.
- slots are of implemented as (offset, length)
Latching summary:
Each page has an associated read/write lock. This lock only
protects the internal layout of the page, and the members of the
page struct. Here is how it is held in various circumstances:
Record allocation: Write lock
Record read: Read lock
Read LSN Read lock
Record write *READ LOCK*
Write LSN Write lock
Any circumstance where these locks are held during an I/O operation
is a bug.
$Id$
************************************************************************/
#include "../page.h"
#include "../blobManager.h"
#include "slotted.h"
@ -143,6 +100,7 @@ void pageInitialize(Page * page) {
/* printf("Initializing page %d\n", page->id);
fflush(NULL); */
memset(page->memAddr, 0, PAGE_SIZE);
*page_type_ptr(page) = SLOTTED_PAGE;
*freespace_ptr(page) = 0;
*numslots_ptr(page) = 0;
*freelist_ptr(page) = INVALID_SLOT;

60
src/lladd/page/slotted.h
View file

@ -1,3 +1,63 @@
/************************************************************************
* @file implementation of variable-sized slotted pages
STRUCTURE OF A PAGE
<pre>
+-----------------------------------------+-----------------------+----+
| DATA SECTION +--------->| RID: (PAGE, 0) | |
| +-----------------+ | +-----------------------+ |
| +-->| RID: (PAGE, 1) | | |
| | +-----------------+ | |
| | | |
| ----------------+ | +------------------------------+
| | | +--->| RID: (PAGE, n) |
| | | | +------------------------------+
|======================================================================|
| ^ FREE SPACE | | | |
| | | | | |
| +-------------------|-------|---|--------------------+ |
| | | | | |
| +-------------|-------|---+ | |
| | | | | |
| +---|---+-----+---|---+---|---+--------------+-----|------+-------+
| | slotn | ... | slot1 | slot0 | num of slots | free space | *** |
+----+-------+-----+-------+-------+--------------+------------+-------+
</pre>
*** = @see page.h for information on this field.
NOTE:
- slots are zero indexed.
- slots are of implemented as (offset, length)
Slotted page layout:
END:
lsn (4 bytes)
type (2 bytes)
free space (2 bytes)
num of slots (2 bytes)
freelist head(2 bytes)
slot 0 (4 bytes)
slot 1 (4 bytes)
...
slot n (4 bytes)
...
unused
...
record n (x bytes)
...
record 0 (y bytes)
record 1 (z bytes)
START
$Id$
************************************************************************/
#define SLOTTED_PAGE_OVERHEAD_PER_RECORD 4
#define SLOTTED_PAGE_HEADER_OVERHEAD 6
void pageWriteRecord(int xid, Page * page, lsn_t lsn, recordid rid, const byte *data);
void pageReadRecord(int xid, Page * page, recordid rid, byte *buff);

4
test/lladd/Makefile.am
View file

@ -1,11 +1,11 @@
INCLUDES = @CHECK_CFLAGS@
if HAVE_CHECK
## Had to disable check_lht because lht needs to be rewritten.
TESTS = check_logEntry check_logWriter check_page check_operations check_transactional2 check_recovery check_blobRecovery check_bufferManager
TESTS = check_logEntry check_logWriter check_page check_operations check_transactional2 check_recovery check_blobRecovery check_bufferManager check_indirect
else
TESTS =
endif
noinst_PROGRAMS = $(TESTS)
LDADD = @CHECK_LIBS@ $(top_builddir)/src/lladd/liblladd.a $(top_builddir)/src/pbl/libpbl.a $(top_builddir)/src/libdfa/librw.a #-lefence
CLEANFILES = check_lht.log check_logEntry.log storefile.txt logfile.txt blob0_file.txt blob1_file.txt check_blobRecovery.log check_logWriter.log check_operations.log check_recovery.log check_transactional2.log check_page.log check_bufferManager.log
CLEANFILES = check_lht.log check_logEntry.log storefile.txt logfile.txt blob0_file.txt blob1_file.txt check_blobRecovery.log check_logWriter.log check_operations.log check_recovery.log check_transactional2.log check_page.log check_bufferManager.log check_indirect.log
AM_CFLAGS= -g -Wall -pedantic -std=gnu99