From: teodor <teodor>
Date: Thu, 19 Oct 2006 13:04:54 +0000 (+0000)
Subject: Add common c-utils, make flatdb aligned on disk
X-Git-Url: http://www.sigaev.ru/git/gitweb.cgi?p=tedtools.git;a=commitdiff_plain;h=616042000fa85135606f626940ca1c73d3a58b36

Add common c-utils, make flatdb aligned on disk
---

diff --git a/expected/flatdb b/expected/flatdb
index 2bdea6c..29e3016 100644
--- a/expected/flatdb
+++ b/expected/flatdb
@@ -1,150 +1,128 @@
 Put: off:16 len:4 'ariz'
 Put: off:24 len:6 'ddk29f'
-Put: off:34 len:1 's'
-Put: off:39 len:4 'a99t'
-Put: off:47 len:7 'suf8eee'
-Put: off:58 len:4 'z0kv'
-Put: off:66 len:3 'lw2'
-Put: off:73 len:4 'twv4'
-Put: off:81 len:4 '8co6'
-Put: off:89 len:9 'u2nie4xqc'
-Put: off:102 len:3 'qgw'
-Put: off:109 len:6 'cnpebn'
-Put: off:119 len:10 'mmlkvhe6ox'
-Put: off:133 len:4 'qite'
-Put: off:141 len:9 'nqzpq8aqj'
-Put: off:154 len:9 'ill7niztn'
-Put: off:167 len:9 'khpq87qew'
-Put: off:180 len:1 'q'
-Put: off:185 len:2 'wz'
-Put: off:191 len:5 'wgrmy'
-Put: off:200 len:8 '8ch2icos'
-Put: off:212 len:4 'v4xc'
-Put: off:220 len:10 'ljklkyz5ud'
-Put: off:234 len:6 '72jzxf'
-Put: off:244 len:7 'r4ysgda'
-Put: off:255 len:2 '9r'
-Put: off:261 len:1 'x'
-Put: off:266 len:5 '7o8gh'
-Put: off:275 len:2 'yi'
-Put: off:281 len:10 'oxr8qbhgwl'
-Put: off:295 len:6 'y4zkee'
-Put: off:305 len:4 'yjin'
-Put: off:313 len:5 'ycblv'
-Put: off:322 len:1 'q'
-Put: off:327 len:9 'nlbzzooco'
-Put: off:340 len:4 '1flp'
-Put: off:348 len:5 'xphng'
-Put: off:357 len:10 'avjrii116x'
-Put: off:371 len:9 'fyuxgehv4'
-Put: off:384 len:2 'ia'
+Put: off:36 len:1 's'
+Put: off:44 len:4 'a99t'
+Put: off:52 len:7 'suf8eee'
+Put: off:64 len:4 'z0kv'
+Put: off:72 len:3 'lw2'
+Put: off:80 len:4 'twv4'
+Put: off:88 len:4 '8co6'
+Put: off:96 len:9 'u2nie4xqc'
+Put: off:112 len:3 'qgw'
+Put: off:120 len:6 'cnpebn'
+Put: off:132 len:10 'mmlkvhe6ox'
+Put: off:148 len:4 'qite'
+Put: off:156 len:9 'nqzpq8aqj'
+Put: off:172 len:9 'ill7niztn'
+Put: off:188 len:9 'khpq87qew'
+Put: off:204 len:1 'q'
+Put: off:212 len:2 'wz'
+Put: off:220 len:5 'wgrmy'
+Put: off:232 len:8 '8ch2icos'
+Put: off:244 len:4 'v4xc'
+Put: off:252 len:10 'ljklkyz5ud'
+Put: off:268 len:6 '72jzxf'
+Put: off:280 len:7 'r4ysgda'
+Put: off:292 len:2 '9r'
+Put: off:300 len:1 'x'
+Put: off:308 len:5 '7o8gh'
+Put: off:320 len:2 'yi'
+Put: off:328 len:10 'oxr8qbhgwl'
+Put: off:344 len:6 'y4zkee'
+Put: off:356 len:4 'yjin'
+Put: off:364 len:5 'ycblv'
+Put: off:376 len:1 'q'
+Put: off:384 len:9 'nlbzzooco'
+Put: off:400 len:4 '1flp'
+Put: off:408 len:5 'xphng'
+Put: off:420 len:10 'avjrii116x'
+Put: off:436 len:9 'fyuxgehv4'
+Put: off:452 len:2 'ia'
 Get: off:16 len:4 'ariz'
-Get: off:322 len:1 'q'
-Get: off:340 len:4 '1flp'
-Get: off:348 len:5 'xphng'
 Get: off:24 len:6 'ddk29f'
-Get: off:34 len:1 's'
-Get: off:39 len:4 'a99t'
-Get: off:47 len:7 'suf8eee'
-Get: off:58 len:4 'z0kv'
-Get: off:234 len:6 '72jzxf'
-Get: off:244 len:7 'r4ysgda'
-Get: off:119 len:10 'mmlkvhe6ox'
-Get: off:133 len:4 'qite'
-Get: off:141 len:9 'nqzpq8aqj'
-Get: off:154 len:9 'ill7niztn'
-Get: off:167 len:9 'khpq87qew'
-Get: off:34 len:1 's'
-Get: off:39 len:4 'a99t'
-Get: off:47 len:7 'suf8eee'
-Get: off:58 len:4 'z0kv'
-Get: off:234 len:6 '72jzxf'
-Get: off:244 len:7 'r4ysgda'
-Get: off:119 len:10 'mmlkvhe6ox'
-Get: off:180 len:1 'q'
-Get: off:185 len:2 'wz'
-Get: off:191 len:5 'wgrmy'
-Get: off:200 len:8 '8ch2icos'
-Get: off:327 len:9 'nlbzzooco'
-Get: off:212 len:4 'v4xc'
-Get: off:220 len:10 'ljklkyz5ud'
-Get: off:266 len:5 '7o8gh'
-Get: off:255 len:2 '9r'
-Get: off:261 len:1 'x'
-Get: off:66 len:3 'lw2'
-Get: off:180 len:1 'q'
-Get: off:185 len:2 'wz'
-Get: off:191 len:5 'wgrmy'
-Get: off:200 len:8 '8ch2icos'
-Get: off:327 len:9 'nlbzzooco'
-Get: off:73 len:4 'twv4'
-Get: off:81 len:4 '8co6'
-Get: off:89 len:9 'u2nie4xqc'
-Get: off:102 len:3 'qgw'
-Get: off:109 len:6 'cnpebn'
-Get: off:275 len:2 'yi'
-Get: off:281 len:10 'oxr8qbhgwl'
-Get: off:295 len:6 'y4zkee'
-Get: off:305 len:4 'yjin'
-Get: off:313 len:5 'ycblv'
-Get: off:357 len:10 'avjrii116x'
-Get: off:371 len:9 'fyuxgehv4'
-Get: off:384 len:2 'ia'
+Get: off:36 len:1 's'
+Get: off:44 len:4 'a99t'
+Get: off:52 len:7 'suf8eee'
+Get: off:64 len:4 'z0kv'
+Get: off:72 len:3 'lw2'
+Get: off:80 len:4 'twv4'
+Get: off:88 len:4 '8co6'
+Get: off:96 len:9 'u2nie4xqc'
+Get: off:112 len:3 'qgw'
+Get: off:120 len:6 'cnpebn'
+Get: off:132 len:10 'mmlkvhe6ox'
+Get: off:148 len:4 'qite'
+Get: off:156 len:9 'nqzpq8aqj'
+Get: off:172 len:9 'ill7niztn'
+Get: off:188 len:9 'khpq87qew'
+Get: off:204 len:1 'q'
+Get: off:212 len:2 'wz'
+Get: off:220 len:5 'wgrmy'
+Get: off:232 len:8 '8ch2icos'
+Get: off:244 len:4 'v4xc'
+Get: off:252 len:10 'ljklkyz5ud'
+Get: off:268 len:6 '72jzxf'
+Get: off:280 len:7 'r4ysgda'
+Get: off:292 len:2 '9r'
+Get: off:300 len:1 'x'
+Get: off:308 len:5 '7o8gh'
+Get: off:320 len:2 'yi'
+Get: off:328 len:10 'oxr8qbhgwl'
+Get: off:344 len:6 'y4zkee'
+Get: off:356 len:4 'yjin'
+Get: off:364 len:5 'ycblv'
+Get: off:376 len:1 'q'
+Get: off:384 len:9 'nlbzzooco'
+Get: off:400 len:4 '1flp'
+Get: off:408 len:5 'xphng'
+Get: off:420 len:10 'avjrii116x'
+Get: off:436 len:9 'fyuxgehv4'
+Get: off:452 len:2 'ia'
 Del: off:16
 Del: off:24
-Del: off:34
-Del: off:266
-Del: off:47
-Del: off:185
+Del: off:36
+Del: off:308
+Del: off:52
+Del: off:212
 Vacuum
-Get: off:322 len:1 'q'
-Get: off:340 len:4 '1flp'
-Get: off:348 len:5 'xphng'
-Get: off:39 len:4 'a99t'
-Get: off:58 len:4 'z0kv'
-Get: off:234 len:6 '72jzxf'
-Get: off:244 len:7 'r4ysgda'
-Get: off:119 len:10 'mmlkvhe6ox'
-Get: off:133 len:4 'qite'
-Get: off:141 len:9 'nqzpq8aqj'
-Get: off:154 len:9 'ill7niztn'
-Get: off:167 len:9 'khpq87qew'
-Get: off:34 len:1 's'
-Get: off:39 len:4 'a99t'
-Get: off:58 len:4 'z0kv'
-Get: off:234 len:6 '72jzxf'
-Get: off:244 len:7 'r4ysgda'
-Get: off:119 len:10 'mmlkvhe6ox'
-Get: off:180 len:1 'q'
-Get: off:191 len:5 'wgrmy'
-Get: off:200 len:8 '8ch2icos'
-Get: off:327 len:9 'nlbzzooco'
-Get: off:212 len:4 'v4xc'
-Get: off:220 len:10 'ljklkyz5ud'
-Get: off:255 len:2 '9r'
-Get: off:261 len:1 'x'
-Get: off:66 len:3 'lw2'
-Get: off:180 len:1 'q'
-Get: off:185 len:2 'wz'
-Get: off:191 len:5 'wgrmy'
-Get: off:200 len:8 '8ch2icos'
-Get: off:327 len:9 'nlbzzooco'
-Get: off:73 len:4 'twv4'
-Get: off:81 len:4 '8co6'
-Get: off:89 len:9 'u2nie4xqc'
-Get: off:102 len:3 'qgw'
-Get: off:109 len:6 'cnpebn'
-Get: off:275 len:2 'yi'
-Get: off:281 len:10 'oxr8qbhgwl'
-Get: off:295 len:6 'y4zkee'
-Get: off:305 len:4 'yjin'
-Get: off:313 len:5 'ycblv'
-Get: off:357 len:10 'avjrii116x'
-Get: off:371 len:9 'fyuxgehv4'
-Get: off:384 len:2 'ia'
+Get: off:36 len:1 's'
+Get: off:44 len:4 'a99t'
+Get: off:64 len:4 'z0kv'
+Get: off:72 len:3 'lw2'
+Get: off:80 len:4 'twv4'
+Get: off:88 len:4 '8co6'
+Get: off:96 len:9 'u2nie4xqc'
+Get: off:112 len:3 'qgw'
+Get: off:120 len:6 'cnpebn'
+Get: off:132 len:10 'mmlkvhe6ox'
+Get: off:148 len:4 'qite'
+Get: off:156 len:9 'nqzpq8aqj'
+Get: off:172 len:9 'ill7niztn'
+Get: off:188 len:9 'khpq87qew'
+Get: off:204 len:1 'q'
+Get: off:220 len:5 'wgrmy'
+Get: off:232 len:8 '8ch2icos'
+Get: off:244 len:4 'v4xc'
+Get: off:252 len:10 'ljklkyz5ud'
+Get: off:268 len:6 '72jzxf'
+Get: off:280 len:7 'r4ysgda'
+Get: off:292 len:2 '9r'
+Get: off:300 len:1 'x'
+Get: off:320 len:2 'yi'
+Get: off:328 len:10 'oxr8qbhgwl'
+Get: off:344 len:6 'y4zkee'
+Get: off:356 len:4 'yjin'
+Get: off:364 len:5 'ycblv'
+Get: off:376 len:1 'q'
+Get: off:384 len:9 'nlbzzooco'
+Get: off:400 len:4 '1flp'
+Get: off:408 len:5 'xphng'
+Get: off:420 len:10 'avjrii116x'
+Get: off:436 len:9 'fyuxgehv4'
+Get: off:452 len:2 'ia'
 List of free space:
-	off: 16	 len: 23
-	off: 47	 len: 11
-	off: 185	 len: 6
-	off: 266	 len: 9
-	off: 390	 len: 60
+	off: 16	 len: 28
+	off: 52	 len: 12
+	off: 212	 len: 8
+	off: 308	 len: 12
+	off: 460	 len: 60
diff --git a/flatdb.c b/flatdb.c
index 56d78eb..10b3848 100644
--- a/flatdb.c
+++ b/flatdb.c
@@ -210,6 +210,7 @@ FDBClose(FDB *db) {
 			FDBVacuumFreeSpace(db);
 
 			header.lenfreespace = sizeof(FDBFreeSpace)*db->listcur;
+
 			ptr = findFreeSpace( db, header.lenfreespace );
 		
 			if ( ptr ) {
@@ -265,7 +266,7 @@ FDBDelete(FDB *db, off_t offset, size_t length) {
 		if ( readLen(db, offset, &length) != FDB_OK )
 			return FDB_ERROR;
 
-	addFreeSpace(db, offset, length);
+	addFreeSpace(db, offset, PTRALIGN(length));
 
 	return FDB_OK;
 }
@@ -311,11 +312,14 @@ FDBGet(FDB *db, off_t offset, size_t length, FDBRecord **record) {
 int 
 FDBPut(FDB *db, FDBRecord *record, off_t *offset ) {
 	FDBFreeSpace *ptr;
+	size_t	aligned;
 
 	if ( db->readonly )
 		return FDB_ERROR;
 
-	ptr = findFreeSpace( db, record->length ); 
+	aligned = PTRALIGN(record->length);
+
+	ptr = findFreeSpace( db, aligned ); 
 	if ( ptr ) {
 		*offset = ptr->offset;
 		ptr->length -= record->length;
@@ -334,9 +338,13 @@ FDBPut(FDB *db, FDBRecord *record, off_t *offset ) {
 
 	if ( write(db->fd, record, record->length) != record->length ) 
 		tlog(TL_CRIT|TL_EXIT,"FDBPut: write failed: %s", strerror(errno));
+	if ( record->length != aligned ) {
+		char	buf[] = {0, 0, 0, 0, 0, 0, 0, 0};
+		if ( write(db->fd, buf, aligned - record->length) != (aligned - record->length) ) 
+			tlog(TL_CRIT|TL_EXIT,"FDBPut: write failed: %s", strerror(errno));
+	}
 
 	return FDB_OK;
-
 }
 
 
diff --git a/flatdb.h b/flatdb.h
index 5c3a4da..c94618b 100644
--- a/flatdb.h
+++ b/flatdb.h
@@ -30,6 +30,7 @@
 #define __FLAT_DB__
 
 #include <sys/types.h>
+#include "tools.h"
 
 typedef struct {
 	off_t	freespace;
@@ -44,7 +45,7 @@ typedef struct {
 	char	data[1];
 } FDBRecord;
 
-#define RECHDRSZ	(sizeof(size_t))
+#define RECHDRSZ	(offsetof(FDBRecord, data))
 
 typedef struct {
 	off_t	offset;
diff --git a/tbtree.h b/tbtree.h
index 3971fda..28d8837 100644
--- a/tbtree.h
+++ b/tbtree.h
@@ -32,11 +32,7 @@
 
 
 #include <sys/types.h>
-
-/* C-utils */
-#ifndef offsetof
-#define offsetof(type, field)   ((int) &((type *)0)->field)
-#endif   /* offsetof */
+#include "tools.h" 
 
 #define HASHSIZE(LEN)	( (LEN)<<1 )
 /* end utils */
diff --git a/tests/flatdb b/tests/flatdb
index 3f111c1..02f330e 100644
--- a/tests/flatdb
+++ b/tests/flatdb
@@ -1,12 +1,14 @@
 [ -f temp/FLAT ] && rm -rf temp/FLAT 
 ./flatdbtest -f temp/FLAT -p ariz -p ddk29f -p s -p a99t -p suf8eee -p z0kv -p lw2 -p twv4 -p 8co6 -p u2nie4xqc -p qgw -p cnpebn -p mmlkvhe6ox -p qite -p nqzpq8aqj -p ill7niztn -p khpq87qew -p q -p wz -p wgrmy -p 8ch2icos -p v4xc -p ljklkyz5ud -p 72jzxf -p r4ysgda -p 9r -p x -p 7o8gh -p yi -p oxr8qbhgwl -p y4zkee -p yjin -p ycblv -p q -p nlbzzooco -p 1flp -p xphng -p avjrii116x -p fyuxgehv4 -p ia || exit 1
 
-./flatdbtest -r -f temp/FLAT -g 16 -g 322 -g 340 -g 348 -g 24 -g 34 -g 39 -g 47 -g 58 -g 234 -g 244 -g 119 -g 133 -g 141 -g 154 -g 167 -g 34 -g 39 -g 47 -g 58 -g 234 -g 244 -g 119 -g 180 -g 185 -g 191 -g 200 -g 327 -g 212 -g 220 -g 266 -g 255 -g 261 -g 66 -g 180 -g 185 -g 191 -g 200 -g 327 -g 73 -g 81 -g 89 -g 102 -g 109 -g 275 -g 281 -g 295 -g 305 -g 313 -g 357 -g 371 -g 384 || exit 1
+./flatdbtest -r -f temp/FLAT -g 16 -g 24 -g 36 -g 44 -g 52 -g 64 -g 72 -g 80 -g 88 -g 96 -g 112 -g 120 -g 132 -g 148 -g 156 -g 172 -g 188 -g 204 -g 212 -g 220 -g 232 -g 244 -g 252 -g 268 -g 280 -g 292 -g 300 -g 308 -g 320 -g 328 -g 344 -g 356 -g 364 -g 376 -g 384 -g 400 -g 408 -g 420 -g 436 -g 452 || exit 1
 
-./flatdbtest -f temp/FLAT -d 16 -d 24 -d 34 -d 266 -d 47 -d 185 || exit 1
+./flatdbtest -f temp/FLAT -d 16 -d 24 -d 36 -d 308 -d 52 -d 212 || exit 1
 
 ./flatdbtest -f temp/FLAT -v || exit 1 
 
-./flatdbtest -r -f temp/FLAT -g 322 -g 340 -g 348 -g 39 -g 58 -g 234 -g 244 -g 119 -g 133 -g 141 -g 154 -g 167 -g 34 -g 39 -g 58 -g 234 -g 244 -g 119 -g 180 -g 191 -g 200 -g 327 -g 212 -g 220 -g 255 -g 261 -g 66 -g 180 -g 185 -g 191 -g 200 -g 327 -g 73 -g 81 -g 89 -g 102 -g 109 -g 275 -g 281 -g 295 -g 305 -g 313 -g 357 -g 371 -g 384 || exit 1
+./flatdbtest -r -f temp/FLAT -g 36 -g 44 -g 64 -g 72 -g 80 -g 88 -g 96 -g 112 -g 120 -g 132 -g 148 -g 156 -g 172 -g 188 -g 204 -g 220 -g 232 -g 244 -g 252 -g 268 -g 280 -g 292 -g 300 -g 320 -g 328 -g 344 -g 356 -g 364 -g 376 -g 384 -g 400 -g 408 -g 420 -g 436 -g 452 || exit 1
 
 ./flatdbtest -f temp/FLAT -s || exit 1
+
+
diff --git a/tmalloc.h b/tmalloc.h
index fce5dba..a39a3ff 100644
--- a/tmalloc.h
+++ b/tmalloc.h
@@ -31,6 +31,7 @@
 #define __TMALLOC_H__
 
 #include <sys/types.h>
+#include "tools.h"
 
 void * tmalloc(size_t size);
 void * trealloc(void * ptr, size_t size); 
@@ -74,11 +75,6 @@ typedef struct {
 #define MCASHDRSZ ( sizeof(size_t) + sizeof(MemoryContext*) )
 #define MCMAGICKNUMBER	(0xFE0FBEEF)
 
-#define TYPEALIGN(ALIGNVAL,LEN)  \
-        (((long) (LEN) + (ALIGNVAL-1)) & ~((long) (ALIGNVAL-1)))
-
-#define PTRALIGN(LEN)     TYPEALIGN(sizeof(void*), (LEN))
-
 MemoryContext *allocMemoryContext(MemoryContext* parent, int flags);
 void freeMemoryContext(MemoryContext* cntx);
 void*   mcalloc(MemoryContext *cntx, size_t size);
diff --git a/tools.h b/tools.h
index d70d14d..6db856d 100644
--- a/tools.h
+++ b/tools.h
@@ -39,4 +39,19 @@ char*     xtostr(u_int32_t x);
 
 double timediff(struct timeval *begin, struct timeval *end);
 double elapsedtime(struct timeval *begin);
+
+#ifndef offsetof
+#define offsetof(type, field)   ((int) &((type *)0)->field)
+#endif   /* offsetof */
+
+#ifndef TYPEALIGN
+#define TYPEALIGN(ALIGNVAL,LEN)  \
+        (((long) (LEN) + ((ALIGNVAL) - 1)) & ~((long) ((ALIGNVAL) - 1)))
+
+#define SHORTALIGN(LEN)                 TYPEALIGN(sizeof(int16), (LEN))
+#define INTALIGN(LEN)                   TYPEALIGN(sizeof(int32), (LEN))
+#define MAXALIGN(LEN)                   TYPEALIGN(sizeof(int64), (LEN))
+#define PTRALIGN(LEN)                   TYPEALIGN(sizeof(void*), (LEN))
+#endif
+
 #endif