File D:\maven-qcr-plugin-testsunitaires\qcr4java\h2-1.3.170-sources\target\site\qc\lecode\store\PageStore.java.html

1 /*
2 * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
3 * Version 1.0, and under the Eclipse Public License, Version 1.0
4 * (http://h2database.com/html/license.html).
5 * Initial Developer: H2 Group
6 */
7 package org.h2.store;
8
9 import java.io.IOException;
10 import java.io.OutputStream;
11 import java.util.ArrayList;
12 import java.util.Collections;
13 import java.util.HashMap;
14 import java.util.zip.CRC32;
15 import org.h2.command.CommandInterface;
16 import org.h2.command.ddl.CreateTableData;
17 import org.h2.constant.ErrorCode;
18 import org.h2.constant.SysProperties;
19 import org.h2.engine.Constants;
20 import org.h2.engine.Database;
21 import org.h2.engine.Session;
22 import org.h2.index.Cursor;
23 import org.h2.index.Index;
24 import org.h2.index.IndexType;
25 import org.h2.index.MultiVersionIndex;
26 import org.h2.index.PageBtreeIndex;
27 import org.h2.index.PageBtreeLeaf;
28 import org.h2.index.PageBtreeNode;
29 import org.h2.index.PageDataIndex;
30 import org.h2.index.PageDataLeaf;
31 import org.h2.index.PageDataNode;
32 import org.h2.index.PageDataOverflow;
33 import org.h2.index.PageDelegateIndex;
34 import org.h2.index.PageIndex;
35 import org.h2.message.DbException;
36 import org.h2.message.Trace;
37 import org.h2.result.Row;
38 import org.h2.schema.Schema;
39 import org.h2.store.fs.FileUtils;
40 import org.h2.table.Column;
41 import org.h2.table.IndexColumn;
42 import org.h2.table.RegularTable;
43 import org.h2.table.Table;
44 import org.h2.util.BitField;
45 import org.h2.util.Cache;
46 import org.h2.util.CacheLRU;
47 import org.h2.util.CacheObject;
48 import org.h2.util.CacheWriter;
49 import org.h2.util.IntArray;
50 import org.h2.util.IntIntHashMap;
51 import org.h2.util.New;
52 import org.h2.util.StatementBuilder;
53 import org.h2.util.StringUtils;
54 import org.h2.value.CompareMode;
55 import org.h2.value.Value;
56 import org.h2.value.ValueInt;
57 import org.h2.value.ValueString;
58
59 /**
60 * This class represents a file that is organized as a number of pages. Page 0
61 * contains a static file header, and pages 1 and 2 both contain the variable
62 * file header (page 2 is a copy of page 1 and is only read if the checksum of
63 * page 1 is invalid). The format of page 0 is:
64 * <ul>
65 * <li>0-47: file header (3 time "-- H2 0.5/B -- \n")</li>
66 * <li>48-51: page size in bytes (512 - 32768, must be a power of 2)</li>
67 * <li>52: write version (read-only if larger than 1)</li>
68 * <li>53: read version (opening fails if larger than 1)</li>
69 * </ul>
70 * The format of page 1 and 2 is:
71 * <ul>
72 * <li>CRC32 of the remaining data: int (0-3)</li>
73 * <li>write counter (incremented on each write): long (4-11)</li>
74 * <li>log trunk key: int (12-15)</li>
75 * <li>log trunk page (0 for none): int (16-19)</li>
76 * <li>log data page (0 for none): int (20-23)</li>
77 * </ul>
78 * Page 3 contains the first free list page.
79 * Page 4 contains the meta table root page.
80 */
81 public class PageStore implements CacheWriter {
82
83 // TODO test running out of disk space (using a special file system)
84 // TODO unused pages should be freed once in a while
85 // TODO node row counts are incorrect (it's not splitting row counts)
86 // TODO after opening the database, delay writing until required
87 // TODO optimization: try to avoid allocating a byte array per page
88 // TODO optimization: check if calling Data.getValueLen slows things down
89 // TODO order pages so that searching for a key only seeks forward
90 // TODO optimization: update: only log the key and changed values
91 // TODO index creation: use less space (ordered, split at insertion point)
92 // TODO detect circles in linked lists
93 // (input stream, free list, extend pages...)
94 // at runtime and recovery
95 // TODO remove trace or use isDebugEnabled
96 // TODO recover tool: support syntax to delete a row with a key
97 // TODO don't store default values (store a special value)
98 // TODO check for file size (exception if not exact size expected)
99 // TODO online backup using bsdiff
100
101 /**
102 * The smallest possible page size.
103 */
104 public static final int PAGE_SIZE_MIN = 64;
105
106 /**
107 * The biggest possible page size.
108 */
109 public static final int PAGE_SIZE_MAX = 32768;
110
111 /**
112 * This log mode means the transaction log is not used.
113 */
114 public static final int LOG_MODE_OFF = 0;
115
116 /**
117 * This log mode means the transaction log is used and FileDescriptor.sync()
118 * is called for each checkpoint. This is the default level.
119 */
120 public static final int LOG_MODE_SYNC = 2;
121 private static final int PAGE_ID_FREE_LIST_ROOT = 3;
122 private static final int PAGE_ID_META_ROOT = 4;
123 private static final int MIN_PAGE_COUNT = 6;
124 private static final int INCREMENT_KB = 1024;
125 private static final int INCREMENT_PERCENT_MIN = 35;
126 private static final int READ_VERSION = 3;
127 private static final int WRITE_VERSION = 3;
128 private static final int META_TYPE_DATA_INDEX = 0;
129 private static final int META_TYPE_BTREE_INDEX = 1;
130 private static final int META_TABLE_ID = -1;
131 private static final int COMPACT_BLOCK_SIZE = 1536;
132 private final Database database;
133 private final Trace trace;
134 private final String fileName;
135 private FileStore file;
136 private String accessMode;
137 private int pageSize = Constants.DEFAULT_PAGE_SIZE;
138 private int pageSizeShift;
139 private long writeCountBase, writeCount, readCount;
140 private int logKey, logFirstTrunkPage, logFirstDataPage;
141 private final Cache cache;
142 private int freeListPagesPerList;
143 private boolean recoveryRunning;
144 private boolean ignoreBigLog;
145
146 /**
147 * The index to the first free-list page that potentially has free space.
148 */
149 private int firstFreeListIndex;
150
151 /**
152 * The file size in bytes.
153 */
154 private long fileLength;
155
156 /**
157 * Number of pages (including free pages).
158 */
159 private int pageCount;
160
161 private PageLog log;
162 private Schema metaSchema;
163 private RegularTable metaTable;
164 private PageDataIndex metaIndex;
165 private final IntIntHashMap metaRootPageId = new IntIntHashMap();
166 private final HashMap<Integer, PageIndex> metaObjects = New.hashMap();
167 private HashMap<Integer, PageIndex> tempObjects;
168
169 /**
170 * The map of reserved pages, to ensure index head pages
171 * are not used for regular data during recovery. The key is the page id,
172 * and the value the latest transaction position where this page is used.
173 */
174 private HashMap<Integer, Integer> reservedPages;
175 private boolean isNew;
176 private long maxLogSize = Constants.DEFAULT_MAX_LOG_SIZE;
177 private final Session systemSession;
178
179 /**
180 * Each free page is marked with a set bit.
181 */
182 private final BitField freed = new BitField();
183 private final ArrayList<PageFreeList> freeLists = New.arrayList();
184
185 private boolean recordPageReads;
186 private ArrayList<Integer> recordedPagesList;
187 private IntIntHashMap recordedPagesIndex;
188
189 /**
190 * The change count is something like a "micro-transaction-id".
191 * It is used to ensure that changed pages are not written to the file
192 * before the the current operation is not finished. This is only a problem
193 * when using a very small cache size. The value starts at 1 so that
194 * pages with change count 0 can be evicted from the cache.
195 */
196 private int changeCount = 1;
197
198 private Data emptyPage;
199 private long logSizeBase;
200 private HashMap<String, Integer> statistics;
201 private int logMode = LOG_MODE_SYNC;
202 private boolean lockFile;
203 private boolean readMode;
204 private int backupLevel;
205
206 /**
207 * Create a new page store object.
208 *
209 * @param database the database
210 * @param fileName the file name
211 * @param accessMode the access mode
212 * @param cacheSizeDefault the default cache size
213 */
214 public PageStore(Database database, String fileName, String accessMode, int cacheSizeDefault) {
215 this.fileName = fileName;
216 this.accessMode = accessMode;
217 this.database = database;
218 trace = database.getTrace(Trace.PAGE_STORE);
219 // if (fileName.endsWith("X.h2.db"))
220 // trace.setLevel(TraceSystem.DEBUG);
221 String cacheType = database.getCacheType();
222 this.cache = CacheLRU.getCache(this, cacheType, cacheSizeDefault);
223 systemSession = new Session(database, null, 0);
224 }
225
226 /**
227 * Start collecting statistics.
228 */
229 public void statisticsStart() {
230 statistics = New.hashMap();
231 }
232
233 /**
234 * Stop collecting statistics.
235 *
236 * @return the statistics
237 */
238 public HashMap<String, Integer> statisticsEnd() {
239 HashMap<String, Integer> result = statistics;
240 statistics = null;
241 return result;
242 }
243
244 private void statisticsIncrement(String key) {
245 if (statistics != null) {
246 Integer old = statistics.get(key);
247 statistics.put(key, old == null ? 1 : old + 1);
248 }
249 }
250
251 /**
252 * Copy the next page to the output stream.
253 *
254 * @param pageId the page to copy
255 * @param out the output stream
256 * @return the new position, or -1 if there is no more data to copy
257 */
258 public synchronized int copyDirect(int pageId, OutputStream out) throws IOException {
259 byte[] buffer = new byte[pageSize];
260 if (pageId >= pageCount) {
261 return -1;
262 }
263 file.seek((long) pageId << pageSizeShift);
264 file.readFullyDirect(buffer, 0, pageSize);
265 readCount++;
266 out.write(buffer, 0, pageSize);
267 return pageId + 1;
268 }
269
270 /**
271 * Open the file and read the header.
272 */
273 public synchronized void open() {
274 try {
275 metaRootPageId.put(META_TABLE_ID, PAGE_ID_META_ROOT);
276 if (FileUtils.exists(fileName)) {
277 long length = FileUtils.size(fileName);
278 if (length < MIN_PAGE_COUNT * PAGE_SIZE_MIN) {
279 if (database.isReadOnly()) {
280 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, fileName + " length: " + length);
281 }
282 // the database was not fully created
283 openNew();
284 } else {
285 openExisting();
286 }
287 } else {
288 openNew();
289 }
290 } catch (DbException e) {
291 close();
292 throw e;
293 }
294 }
295
296 private void openNew() {
297 setPageSize(pageSize);
298 freeListPagesPerList = PageFreeList.getPagesAddressed(pageSize);
299 file = database.openFile(fileName, accessMode, false);
300 lockFile();
301 recoveryRunning = true;
302 writeStaticHeader();
303 writeVariableHeader();
304 log = new PageLog(this);
305 increaseFileSize(MIN_PAGE_COUNT);
306 openMetaIndex();
307 logFirstTrunkPage = allocatePage();
308 log.openForWriting(logFirstTrunkPage, false);
309 isNew = true;
310 recoveryRunning = false;
311 increaseFileSize();
312 }
313
314 private void lockFile() {
315 if (lockFile) {
316 if (!file.tryLock()) {
317 throw DbException.get(ErrorCode.DATABASE_ALREADY_OPEN_1, fileName);
318 }
319 }
320 }
321
322 private void openExisting() {
323 try {
324 file = database.openFile(fileName, accessMode, true);
325 } catch (DbException e) {
326 if (e.getErrorCode() == ErrorCode.IO_EXCEPTION_2) {
327 if (e.getMessage().indexOf("locked") >= 0) {
328 // in Windows, you can't open a locked file
329 // (in other operating systems, you can)
330 // the exact error message is:
331 // "The process cannot access the file because
332 // another process has locked a portion of the file"
333 throw DbException.get(ErrorCode.DATABASE_ALREADY_OPEN_1, e, fileName);
334 }
335 }
336 throw e;
337 }
338 lockFile();
339 readStaticHeader();
340 freeListPagesPerList = PageFreeList.getPagesAddressed(pageSize);
341 fileLength = file.length();
342 pageCount = (int) (fileLength / pageSize);
343 if (pageCount < MIN_PAGE_COUNT) {
344 if (database.isReadOnly()) {
345 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, fileName + " pageCount: " + pageCount);
346 }
347 file.releaseLock();
348 file.close();
349 FileUtils.delete(fileName);
350 openNew();
351 return;
352 }
353 readVariableHeader();
354 log = new PageLog(this);
355 log.openForReading(logKey, logFirstTrunkPage, logFirstDataPage);
356 boolean old = database.isMultiVersion();
357 // temporarily disabling multi-version concurrency, because
358 // the multi-version index sometimes compares rows
359 // and the LOB storage is not yet available.
360 database.setMultiVersion(false);
361 boolean isEmpty = recover();
362 database.setMultiVersion(old);
363 if (!database.isReadOnly()) {
364 readMode = true;
365 if (!isEmpty || !SysProperties.MODIFY_ON_WRITE || tempObjects != null) {
366 openForWriting();
367 removeOldTempIndexes();
368 }
369 }
370 }
371
372 private void openForWriting() {
373 if (!readMode || database.isReadOnly()) {
374 return;
375 }
376 readMode = false;
377 recoveryRunning = true;
378 log.free();
379 logFirstTrunkPage = allocatePage();
380 log.openForWriting(logFirstTrunkPage, false);
381 recoveryRunning = false;
382 freed.set(0, pageCount, true);
383 checkpoint();
384 }
385
386 private void removeOldTempIndexes() {
387 if (tempObjects != null) {
388 metaObjects.putAll(tempObjects);
389 for (PageIndex index: tempObjects.values()) {
390 if (index.getTable().isTemporary()) {
391 index.truncate(systemSession);
392 index.remove(systemSession);
393 }
394 }
395 systemSession.commit(true);
396 tempObjects = null;
397 }
398 metaObjects.clear();
399 metaObjects.put(-1, metaIndex);
400 }
401
402 private void writeIndexRowCounts() {
403 for (PageIndex index: metaObjects.values()) {
404 index.writeRowCount();
405 }
406 }
407
408 private void writeBack() {
409 ArrayList<CacheObject> list = cache.getAllChanged();
410 Collections.sort(list);
411 for (int i = 0, size = list.size(); i < size; i++) {
412 writeBack(list.get(i));
413 }
414 }
415
416 /**
417 * Flush all pending changes to disk, and switch the new transaction log.
418 */
419 public synchronized void checkpoint() {
420 trace.debug("checkpoint");
421 if (log == null || readMode || database.isReadOnly() || backupLevel > 0) {
422 // the file was never fully opened, or is read-only,
423 // or checkpoint is currently disabled
424 return;
425 }
426 database.checkPowerOff();
427 writeIndexRowCounts();
428
429 log.checkpoint();
430 writeBack();
431
432 int firstUncommittedSection = getFirstUncommittedSection();
433
434 log.removeUntil(firstUncommittedSection);
435
436 // write back the free list
437 writeBack();
438
439 // ensure the free list is backed up again
440 log.checkpoint();
441
442 if (trace.isDebugEnabled()) {
443 trace.debug("writeFree");
444 }
445 byte[] test = new byte[16];
446 byte[] empty = new byte[pageSize];
447 for (int i = PAGE_ID_FREE_LIST_ROOT; i < pageCount; i++) {
448 if (isUsed(i)) {
449 freed.clear(i);
450 } else if (!freed.get(i)) {
451 if (trace.isDebugEnabled()) {
452 trace.debug("free " + i);
453 }
454 file.seek((long) i << pageSizeShift);
455 file.readFully(test, 0, 16);
456 if (test[0] != 0) {
457 file.seek((long) i << pageSizeShift);
458 file.write(empty, 0, pageSize);
459 writeCount++;
460 }
461 freed.set(i);
462 }
463 }
464 }
465
466 /**
467 * Shrink the file so there are no empty pages at the end.
468 *
469 * @param compactMode 0 if no compacting should happen, otherwise
470 * TransactionCommand.SHUTDOWN_COMPACT or TransactionCommand.SHUTDOWN_DEFRAG
471 */
472 public synchronized void compact(int compactMode) {
473 if (!database.getSettings().pageStoreTrim) {
474 return;
475 }
476 if (SysProperties.MODIFY_ON_WRITE && readMode && compactMode == 0) {
477 return;
478 }
479 openForWriting();
480 // find the last used page
481 int lastUsed = -1;
482 for (int i = getFreeListId(pageCount); i >= 0; i--) {
483 lastUsed = getFreeList(i).getLastUsed();
484 if (lastUsed != -1) {
485 break;
486 }
487 }
488 // open a new log at the very end
489 // (to be truncated later)
490 writeBack();
491 log.free();
492 recoveryRunning = true;
493 try {
494 logFirstTrunkPage = lastUsed + 1;
495 allocatePage(logFirstTrunkPage);
496 log.openForWriting(logFirstTrunkPage, true);
497 // ensure the free list is backed up again
498 log.checkpoint();
499 } finally {
500 recoveryRunning = false;
501 }
502 long start = System.currentTimeMillis();
503 boolean isCompactFully = compactMode == CommandInterface.SHUTDOWN_COMPACT;
504 boolean isDefrag = compactMode == CommandInterface.SHUTDOWN_DEFRAG;
505
506 if (database.getSettings().defragAlways) {
507 isCompactFully = isDefrag = true;
508 }
509
510 int maxCompactTime = database.getSettings().maxCompactTime;
511 int maxMove = database.getSettings().maxCompactCount;
512
513 if (isCompactFully || isDefrag) {
514 maxCompactTime = Integer.MAX_VALUE;
515 maxMove = Integer.MAX_VALUE;
516 }
517 int blockSize = isCompactFully ? COMPACT_BLOCK_SIZE : 1;
518 int firstFree = MIN_PAGE_COUNT;
519 for (int x = lastUsed, j = 0; x > MIN_PAGE_COUNT && j < maxMove; x -= blockSize) {
520 for (int full = x - blockSize + 1; full <= x; full++) {
521 if (full > MIN_PAGE_COUNT && isUsed(full)) {
522 synchronized (this) {
523 firstFree = getFirstFree(firstFree);
524 if (firstFree == -1 || firstFree >= full) {
525 j = maxMove;
526 break;
527 }
528 if (compact(full, firstFree)) {
529 j++;
530 long now = System.currentTimeMillis();
531 if (now > start + maxCompactTime) {
532 j = maxMove;
533 break;
534 }
535 }
536 }
537 }
538 }
539 }
540 if (isDefrag) {
541 log.checkpoint();
542 writeBack();
543 cache.clear();
544 ArrayList<Table> tables = database.getAllTablesAndViews(false);
545 recordedPagesList = New.arrayList();
546 recordedPagesIndex = new IntIntHashMap();
547 recordPageReads = true;
548 Session s = database.getSystemSession();
549 for (Table table : tables) {
550 if (!table.isTemporary() && Table.TABLE.equals(table.getTableType())) {
551 Index scanIndex = table.getScanIndex(s);
552 Cursor cursor = scanIndex.find(s, null, null);
553 while (cursor.next()) {
554 cursor.get();
555 }
556 for (Index index : table.getIndexes()) {
557 if (index != scanIndex && index.canScan()) {
558 cursor = index.find(s, null, null);
559 while (cursor.next()) {
560 // the data is already read
561 }
562 }
563 }
564 }
565 }
566 recordPageReads = false;
567 int target = MIN_PAGE_COUNT - 1;
568 int temp = 0;
569 for (int i = 0, size = recordedPagesList.size(); i < size; i++) {
570 log.checkpoint();
571 writeBack();
572 int source = recordedPagesList.get(i);
573 Page pageSource = getPage(source);
574 if (!pageSource.canMove()) {
575 continue;
576 }
577 while (true) {
578 Page pageTarget = getPage(++target);
579 if (pageTarget == null || pageTarget.canMove()) {
580 break;
581 }
582 }
583 if (target == source) {
584 continue;
585 }
586 temp = getFirstFree(temp);
587 if (temp == -1) {
588 DbException.throwInternalError("no free page for defrag");
589 }
590 cache.clear();
591 swap(source, target, temp);
592 int index = recordedPagesIndex.get(target);
593 if (index != IntIntHashMap.NOT_FOUND) {
594 recordedPagesList.set(index, source);
595 recordedPagesIndex.put(source, index);
596 }
597 recordedPagesList.set(i, target);
598 recordedPagesIndex.put(target, i);
599 }
600 recordedPagesList = null;
601 recordedPagesIndex = null;
602 }
603 // TODO can most likely be simplified
604 checkpoint();
605 log.checkpoint();
606 writeIndexRowCounts();
607 log.checkpoint();
608 writeBack();
609 commit(systemSession);
610 writeBack();
611 log.checkpoint();
612
613 log.free();
614 // truncate the log
615 recoveryRunning = true;
616 try {
617 setLogFirstPage(++logKey, 0, 0);
618 } finally {
619 recoveryRunning = false;
620 }
621 writeBack();
622 for (int i = getFreeListId(pageCount); i >= 0; i--) {
623 lastUsed = getFreeList(i).getLastUsed();
624 if (lastUsed != -1) {
625 break;
626 }
627 }
628 int newPageCount = lastUsed + 1;
629 if (newPageCount < pageCount) {
630 freed.set(newPageCount, pageCount, false);
631 }
632 pageCount = newPageCount;
633 // the easiest way to remove superfluous entries
634 freeLists.clear();
635 trace.debug("pageCount: " + pageCount);
636 long newLength = (long) pageCount << pageSizeShift;
637 if (file.length() != newLength) {
638 file.setLength(newLength);
639 writeCount++;
640 }
641 }
642
643 private int getFirstFree(int start) {
644 int free = -1;
645 for (int id = getFreeListId(start); start < pageCount; id++) {
646 free = getFreeList(id).getFirstFree(start);
647 if (free != -1) {
648 break;
649 }
650 }
651 return free;
652 }
653
654 private void swap(int a, int b, int free) {
655 if (a < MIN_PAGE_COUNT || b < MIN_PAGE_COUNT) {
656 System.out.println(isUsed(a) + " " + isUsed(b));
657 DbException.throwInternalError("can't swap " + a + " and " + b);
658 }
659 Page f = (Page) cache.get(free);
660 if (f != null) {
661 DbException.throwInternalError("not free: " + f);
662 }
663 if (trace.isDebugEnabled()) {
664 trace.debug("swap " + a + " and " + b + " via " + free);
665 }
666 Page pageA = null;
667 if (isUsed(a)) {
668 pageA = getPage(a);
669 if (pageA != null) {
670 pageA.moveTo(systemSession, free);
671 }
672 free(a);
673 }
674 if (free != b) {
675 if (isUsed(b)) {
676 Page pageB = getPage(b);
677 if (pageB != null) {
678 pageB.moveTo(systemSession, a);
679 }
680 free(b);
681 }
682 if (pageA != null) {
683 f = getPage(free);
684 if (f != null) {
685 f.moveTo(systemSession, b);
686 }
687 free(free);
688 }
689 }
690 }
691
692 private boolean compact(int full, int free) {
693 if (full < MIN_PAGE_COUNT || free == -1 || free >= full || !isUsed(full)) {
694 return false;
695 }
696 Page f = (Page) cache.get(free);
697 if (f != null) {
698 DbException.throwInternalError("not free: " + f);
699 }
700 Page p = getPage(full);
701 if (p == null) {
702 freePage(full);
703 } else if (p instanceof PageStreamData || p instanceof PageStreamTrunk) {
704 if (p.getPos() < log.getMinPageId()) {
705 // an old transaction log page
706 // probably a leftover from a crash
707 freePage(full);
708 }
709 } else {
710 if (trace.isDebugEnabled()) {
711 trace.debug("move " + p.getPos() + " to " + free);
712 }
713 try {
714 p.moveTo(systemSession, free);
715 } finally {
716 changeCount++;
717 }
718 }
719 return true;
720 }
721
722 /**
723 * Read a page from the store.
724 *
725 * @param pageId the page id
726 * @return the page
727 */
728 public synchronized Page getPage(int pageId) {
729 Page p = (Page) cache.get(pageId);
730 if (p != null) {
731 return p;
732 }
733
734 Data data = createData();
735 readPage(pageId, data);
736 int type = data.readByte();
737 if (type == Page.TYPE_EMPTY) {
738 return null;
739 }
740 data.readShortInt();
741 data.readInt();
742 if (!checksumTest(data.getBytes(), pageId, pageSize)) {
743 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "wrong checksum");
744 }
745 switch (type & ~Page.FLAG_LAST) {
746 case Page.TYPE_FREE_LIST:
747 p = PageFreeList.read(this, data, pageId);
748 break;
749 case Page.TYPE_DATA_LEAF: {
750 int indexId = data.readVarInt();
751 PageIndex idx = metaObjects.get(indexId);
752 if (idx == null) {
753 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "index not found " + indexId);
754 }
755 if (!(idx instanceof PageDataIndex)) {
756 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "not a data index " + indexId + " " + idx);
757 }
758 PageDataIndex index = (PageDataIndex) idx;
759 if (statistics != null) {
760 statisticsIncrement(index.getTable().getName() + "." + index.getName() + " read");
761 }
762 p = PageDataLeaf.read(index, data, pageId);
763 break;
764 }
765 case Page.TYPE_DATA_NODE: {
766 int indexId = data.readVarInt();
767 PageIndex idx = metaObjects.get(indexId);
768 if (idx == null) {
769 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "index not found " + indexId);
770 }
771 if (!(idx instanceof PageDataIndex)) {
772 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "not a data index " + indexId + " " + idx);
773 }
774 PageDataIndex index = (PageDataIndex) idx;
775 if (statistics != null) {
776 statisticsIncrement(index.getTable().getName() + "." + index.getName() + " read");
777 }
778 p = PageDataNode.read(index, data, pageId);
779 break;
780 }
781 case Page.TYPE_DATA_OVERFLOW: {
782 p = PageDataOverflow.read(this, data, pageId);
783 if (statistics != null) {
784 statisticsIncrement("overflow read");
785 }
786 break;
787 }
788 case Page.TYPE_BTREE_LEAF: {
789 int indexId = data.readVarInt();
790 PageIndex idx = metaObjects.get(indexId);
791 if (idx == null) {
792 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "index not found " + indexId);
793 }
794 if (!(idx instanceof PageBtreeIndex)) {
795 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "not a btree index " + indexId + " " + idx);
796 }
797 PageBtreeIndex index = (PageBtreeIndex) idx;
798 if (statistics != null) {
799 statisticsIncrement(index.getTable().getName() + "." + index.getName() + " read");
800 }
801 p = PageBtreeLeaf.read(index, data, pageId);
802 break;
803 }
804 case Page.TYPE_BTREE_NODE: {
805 int indexId = data.readVarInt();
806 PageIndex idx = metaObjects.get(indexId);
807 if (idx == null) {
808 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "index not found " + indexId);
809 }
810 if (!(idx instanceof PageBtreeIndex)) {
811 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "not a btree index " + indexId + " " + idx);
812 }
813 PageBtreeIndex index = (PageBtreeIndex) idx;
814 if (statistics != null) {
815 statisticsIncrement(index.getTable().getName() + "." + index.getName() + " read");
816 }
817 p = PageBtreeNode.read(index, data, pageId);
818 break;
819 }
820 case Page.TYPE_STREAM_TRUNK:
821 p = PageStreamTrunk.read(this, data, pageId);
822 break;
823 case Page.TYPE_STREAM_DATA:
824 p = PageStreamData.read(this, data, pageId);
825 break;
826 default:
827 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "page=" + pageId + " type=" + type);
828 }
829 cache.put(p);
830 return p;
831 }
832
833 private int getFirstUncommittedSection() {
834 trace.debug("getFirstUncommittedSection");
835 Session[] sessions = database.getSessions(true);
836 int firstUncommittedSection = log.getLogSectionId();
837 for (Session session : sessions) {
838 int firstUncommitted = session.getFirstUncommittedLog();
839 if (firstUncommitted != Session.LOG_WRITTEN) {
840 if (firstUncommitted < firstUncommittedSection) {
841 firstUncommittedSection = firstUncommitted;
842 }
843 }
844 }
845 return firstUncommittedSection;
846 }
847
848 private void readStaticHeader() {
849 file.seek(FileStore.HEADER_LENGTH);
850 Data page = Data.create(database, new byte[PAGE_SIZE_MIN - FileStore.HEADER_LENGTH]);
851 file.readFully(page.getBytes(), 0, PAGE_SIZE_MIN - FileStore.HEADER_LENGTH);
852 readCount++;
853 setPageSize(page.readInt());
854 int writeVersion = page.readByte();
855 int readVersion = page.readByte();
856 if (readVersion > READ_VERSION) {
857 throw DbException.get(ErrorCode.FILE_VERSION_ERROR_1, fileName);
858 }
859 if (writeVersion > WRITE_VERSION) {
860 close();
861 database.setReadOnly(true);
862 accessMode = "r";
863 file = database.openFile(fileName, accessMode, true);
864 }
865 }
866
867 private void readVariableHeader() {
868 Data page = createData();
869 for (int i = 1;; i++) {
870 if (i == 3) {
871 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, fileName);
872 }
873 page.reset();
874 readPage(i, page);
875 CRC32 crc = new CRC32();
876 crc.update(page.getBytes(), 4, pageSize - 4);
877 int expected = (int) crc.getValue();
878 int got = page.readInt();
879 if (expected == got) {
880 writeCountBase = page.readLong();
881 logKey = page.readInt();
882 logFirstTrunkPage = page.readInt();
883 logFirstDataPage = page.readInt();
884 break;
885 }
886 }
887 }
888
889 /**
890 * Set the page size. The size must be a power of two. This method must be
891 * called before opening.
892 *
893 * @param size the page size
894 */
895 public void setPageSize(int size) {
896 if (size < PAGE_SIZE_MIN || size > PAGE_SIZE_MAX) {
897 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, fileName + " pageSize: " + size);
898 }
899 boolean good = false;
900 int shift = 0;
901 for (int i = 1; i <= size;) {
902 if (size == i) {
903 good = true;
904 break;
905 }
906 shift++;
907 i += i;
908 }
909 if (!good) {
910 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, fileName);
911 }
912 pageSize = size;
913 emptyPage = createData();
914 pageSizeShift = shift;
915 }
916
917 private void writeStaticHeader() {
918 Data page = Data.create(database, new byte[pageSize - FileStore.HEADER_LENGTH]);
919 page.writeInt(pageSize);
920 page.writeByte((byte) WRITE_VERSION);
921 page.writeByte((byte) READ_VERSION);
922 file.seek(FileStore.HEADER_LENGTH);
923 file.write(page.getBytes(), 0, pageSize - FileStore.HEADER_LENGTH);
924 writeCount++;
925 }
926
927 /**
928 * Set the trunk page and data page id of the log.
929 *
930 * @param logKey the log key of the trunk page
931 * @param trunkPageId the trunk page id
932 * @param dataPageId the data page id
933 */
934 void setLogFirstPage(int logKey, int trunkPageId, int dataPageId) {
935 if (trace.isDebugEnabled()) {
936 trace.debug("setLogFirstPage key: " + logKey + " trunk: "+ trunkPageId +" data: " + dataPageId);
937 }
938 this.logKey = logKey;
939 this.logFirstTrunkPage = trunkPageId;
940 this.logFirstDataPage = dataPageId;
941 writeVariableHeader();
942 }
943
944 private void writeVariableHeader() {
945 trace.debug("writeVariableHeader");
946 if (logMode == LOG_MODE_SYNC) {
947 file.sync();
948 }
949 Data page = createData();
950 page.writeInt(0);
951 page.writeLong(getWriteCountTotal());
952 page.writeInt(logKey);
953 page.writeInt(logFirstTrunkPage);
954 page.writeInt(logFirstDataPage);
955 CRC32 crc = new CRC32();
956 crc.update(page.getBytes(), 4, pageSize - 4);
957 page.setInt(0, (int) crc.getValue());
958 file.seek(pageSize);
959 file.write(page.getBytes(), 0, pageSize);
960 file.seek(pageSize + pageSize);
961 file.write(page.getBytes(), 0, pageSize);
962 // don't increment the write counter, because it was just written
963 }
964
965 /**
966 * Close the file without further writing.
967 */
968 public synchronized void close() {
969 trace.debug("close");
970 if (log != null) {
971 log.close();
972 log = null;
973 }
974 if (file != null) {
975 try {
976 file.releaseLock();
977 file.close();
978 } finally {
979 file = null;
980 }
981 }
982 }
983
984 public synchronized void flushLog() {
985 if (file != null) {
986 log.flush();
987 }
988 }
989
990 /**
991 * Flush the transaction log and sync the file.
992 */
993 public synchronized void sync() {
994 if (file != null) {
995 log.flush();
996 file.sync();
997 }
998 }
999
1000 public Trace getTrace() {
1001 return trace;
1002 }
1003
1004 public synchronized void writeBack(CacheObject obj) {
1005 Page record = (Page) obj;
1006 if (trace.isDebugEnabled()) {
1007 trace.debug("writeBack " + record);
1008 }
1009 record.write();
1010 record.setChanged(false);
1011 }
1012
1013 /**
1014 * Write an undo log entry if required.
1015 *
1016 * @param page the page
1017 * @param old the old data (if known) or null
1018 */
1019 public synchronized void logUndo(Page page, Data old) {
1020 if (logMode == LOG_MODE_OFF) {
1021 return;
1022 }
1023 checkOpen();
1024 database.checkWritingAllowed();
1025 if (!recoveryRunning) {
1026 int pos = page.getPos();
1027 if (!log.getUndo(pos)) {
1028 if (old == null) {
1029 old = readPage(pos);
1030 }
1031 openForWriting();
1032 log.addUndo(pos, old);
1033 }
1034 }
1035 }
1036
1037 /**
1038 * Update a page.
1039 *
1040 * @param page the page
1041 */
1042 public synchronized void update(Page page) {
1043 if (trace.isDebugEnabled()) {
1044 if (!page.isChanged()) {
1045 trace.debug("updateRecord " + page.toString());
1046 }
1047 }
1048 checkOpen();
1049 database.checkWritingAllowed();
1050 page.setChanged(true);
1051 int pos = page.getPos();
1052 if (SysProperties.CHECK && !recoveryRunning) {
1053 // ensure the undo entry is already written
1054 if (logMode != LOG_MODE_OFF) {
1055 log.addUndo(pos, null);
1056 }
1057 }
1058 allocatePage(pos);
1059 cache.update(pos, page);
1060 }
1061
1062 private int getFreeListId(int pageId) {
1063 return (pageId - PAGE_ID_FREE_LIST_ROOT) / freeListPagesPerList;
1064 }
1065
1066 private PageFreeList getFreeListForPage(int pageId) {
1067 return getFreeList(getFreeListId(pageId));
1068 }
1069
1070 private PageFreeList getFreeList(int i) {
1071 PageFreeList list = null;
1072 if (i < freeLists.size()) {
1073 list = freeLists.get(i);
1074 if (list != null) {
1075 return list;
1076 }
1077 }
1078 int p = PAGE_ID_FREE_LIST_ROOT + i * freeListPagesPerList;
1079 while (p >= pageCount) {
1080 increaseFileSize();
1081 }
1082 if (p < pageCount) {
1083 list = (PageFreeList) getPage(p);
1084 }
1085 if (list == null) {
1086 list = PageFreeList.create(this, p);
1087 cache.put(list);
1088 }
1089 while (freeLists.size() <= i) {
1090 freeLists.add(null);
1091 }
1092 freeLists.set(i, list);
1093 return list;
1094 }
1095
1096 private void freePage(int pageId) {
1097 int index = getFreeListId(pageId);
1098 PageFreeList list = getFreeList(index);
1099 firstFreeListIndex = Math.min(index, firstFreeListIndex);
1100 list.free(pageId);
1101 }
1102
1103 /**
1104 * Set the bit of an already allocated page.
1105 *
1106 * @param pageId the page to allocate
1107 */
1108 void allocatePage(int pageId) {
1109 PageFreeList list = getFreeListForPage(pageId);
1110 list.allocate(pageId);
1111 }
1112
1113 private boolean isUsed(int pageId) {
1114 return getFreeListForPage(pageId).isUsed(pageId);
1115 }
1116
1117 /**
1118 * Allocate a number of pages.
1119 *
1120 * @param list the list where to add the allocated pages
1121 * @param pagesToAllocate the number of pages to allocate
1122 * @param exclude the exclude list
1123 * @param after all allocated pages are higher than this page
1124 */
1125 void allocatePages(IntArray list, int pagesToAllocate, BitField exclude, int after) {
1126 list.ensureCapacity(list.size() + pagesToAllocate);
1127 for (int i = 0; i < pagesToAllocate; i++) {
1128 int page = allocatePage(exclude, after);
1129 after = page;
1130 list.add(page);
1131 }
1132 }
1133
1134 /**
1135 * Allocate a page.
1136 *
1137 * @return the page id
1138 */
1139 public synchronized int allocatePage() {
1140 openForWriting();
1141 int pos = allocatePage(null, 0);
1142 if (!recoveryRunning) {
1143 if (logMode != LOG_MODE_OFF) {
1144 log.addUndo(pos, emptyPage);
1145 }
1146 }
1147 return pos;
1148 }
1149
1150 private int allocatePage(BitField exclude, int first) {
1151 int page;
1152 for (int i = firstFreeListIndex;; i++) {
1153 PageFreeList list = getFreeList(i);
1154 page = list.allocate(exclude, first);
1155 if (page >= 0) {
1156 firstFreeListIndex = i;
1157 break;
1158 }
1159 }
1160 while (page >= pageCount) {
1161 increaseFileSize();
1162 }
1163 if (trace.isDebugEnabled()) {
1164 // trace.debug("allocatePage " + pos);
1165 }
1166 return page;
1167 }
1168
1169 private void increaseFileSize() {
1170 int increment = INCREMENT_KB * 1024 / pageSize;
1171 int percent = pageCount * INCREMENT_PERCENT_MIN / 100;
1172 if (increment < percent) {
1173 increment = (1 + (percent / increment)) * increment;
1174 }
1175 int max = database.getSettings().pageStoreMaxGrowth;
1176 if (max < increment) {
1177 increment = max;
1178 }
1179 increaseFileSize(increment);
1180 }
1181
1182 private void increaseFileSize(int increment) {
1183 for (int i = pageCount; i < pageCount + increment; i++) {
1184 freed.set(i);
1185 }
1186 pageCount += increment;
1187 long newLength = (long) pageCount << pageSizeShift;
1188 file.setLength(newLength);
1189 writeCount++;
1190 fileLength = newLength;
1191 }
1192
1193 /**
1194 * Add a page to the free list. The undo log entry must have been written.
1195 *
1196 * @param pageId the page id
1197 */
1198 public synchronized void free(int pageId) {
1199 free(pageId, true);
1200 }
1201
1202 /**
1203 * Add a page to the free list.
1204 *
1205 * @param pageId the page id
1206 * @param undo if the undo record must have been written
1207 */
1208 void free(int pageId, boolean undo) {
1209 if (trace.isDebugEnabled()) {
1210 // trace.debug("free " + pageId + " " + undo);
1211 }
1212 cache.remove(pageId);
1213 if (SysProperties.CHECK && !recoveryRunning && undo) {
1214 // ensure the undo entry is already written
1215 if (logMode != LOG_MODE_OFF) {
1216 log.addUndo(pageId, null);
1217 }
1218 }
1219 freePage(pageId);
1220 if (recoveryRunning) {
1221 writePage(pageId, createData());
1222 if (reservedPages != null && reservedPages.containsKey(pageId)) {
1223 // re-allocate the page if it is used later on again
1224 int latestPos = reservedPages.get(pageId);
1225 if (latestPos > log.getLogPos()) {
1226 allocatePage(pageId);
1227 }
1228 }
1229 }
1230 }
1231
1232 /**
1233 * Add a page to the free list. The page is not used, therefore doesn't need
1234 * to be overwritten.
1235 *
1236 * @param pageId the page id
1237 */
1238 void freeUnused(int pageId) {
1239 if (trace.isDebugEnabled()) {
1240 trace.debug("freeUnused " + pageId);
1241 }
1242 cache.remove(pageId);
1243 freePage(pageId);
1244 freed.set(pageId);
1245 }
1246
1247 /**
1248 * Create a data object.
1249 *
1250 * @return the data page.
1251 */
1252 public Data createData() {
1253 return Data.create(database, new byte[pageSize]);
1254 }
1255
1256 /**
1257 * Read a page.
1258 *
1259 * @param pos the page id
1260 * @return the page
1261 */
1262 public synchronized Data readPage(int pos) {
1263 Data page = createData();
1264 readPage(pos, page);
1265 return page;
1266 }
1267
1268 /**
1269 * Read a page.
1270 *
1271 * @param pos the page id
1272 * @param page the page
1273 */
1274 void readPage(int pos, Data page) {
1275 if (recordPageReads) {
1276 if (pos >= MIN_PAGE_COUNT && recordedPagesIndex.get(pos) == IntIntHashMap.NOT_FOUND) {
1277 recordedPagesIndex.put(pos, recordedPagesList.size());
1278 recordedPagesList.add(pos);
1279 }
1280 }
1281 if (pos < 0 || pos >= pageCount) {
1282 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, pos + " of " + pageCount);
1283 }
1284 file.seek((long) pos << pageSizeShift);
1285 file.readFully(page.getBytes(), 0, pageSize);
1286 readCount++;
1287 }
1288
1289 /**
1290 * Get the page size.
1291 *
1292 * @return the page size
1293 */
1294 public int getPageSize() {
1295 return pageSize;
1296 }
1297
1298 /**
1299 * Get the number of pages (including free pages).
1300 *
1301 * @return the page count
1302 */
1303 public int getPageCount() {
1304 return pageCount;
1305 }
1306
1307 /**
1308 * Write a page.
1309 *
1310 * @param pageId the page id
1311 * @param data the data
1312 */
1313 public synchronized void writePage(int pageId, Data data) {
1314 if (pageId <= 0) {
1315 DbException.throwInternalError("write to page " + pageId);
1316 }
1317 byte[] bytes = data.getBytes();
1318 if (SysProperties.CHECK) {
1319 boolean shouldBeFreeList = (pageId - PAGE_ID_FREE_LIST_ROOT) % freeListPagesPerList == 0;
1320 boolean isFreeList = bytes[0] == Page.TYPE_FREE_LIST;
1321 if (bytes[0] != 0 && shouldBeFreeList != isFreeList) {
1322 throw DbException.throwInternalError();
1323 }
1324 }
1325 checksumSet(bytes, pageId);
1326 file.seek((long) pageId << pageSizeShift);
1327 file.write(bytes, 0, pageSize);
1328 writeCount++;
1329 }
1330
1331 /**
1332 * Remove a page from the cache.
1333 *
1334 * @param pageId the page id
1335 */
1336 public synchronized void removeRecord(int pageId) {
1337 cache.remove(pageId);
1338 }
1339
1340 Database getDatabase() {
1341 return database;
1342 }
1343
1344 /**
1345 * Run recovery.
1346 *
1347 * @return whether the transaction log was empty
1348 */
1349 private boolean recover() {
1350 trace.debug("log recover");
1351 recoveryRunning = true;
1352 boolean isEmpty = true;
1353 isEmpty &= log.recover(PageLog.RECOVERY_STAGE_UNDO);
1354 if (reservedPages != null) {
1355 for (int r : reservedPages.keySet()) {
1356 if (trace.isDebugEnabled()) {
1357 trace.debug("reserve " + r);
1358 }
1359 allocatePage(r);
1360 }
1361 }
1362 isEmpty &= log.recover(PageLog.RECOVERY_STAGE_ALLOCATE);
1363 openMetaIndex();
1364 readMetaData();
1365 isEmpty &= log.recover(PageLog.RECOVERY_STAGE_REDO);
1366 boolean setReadOnly = false;
1367 if (!database.isReadOnly()) {
1368 if (log.getInDoubtTransactions().size() == 0) {
1369 log.recoverEnd();
1370 int firstUncommittedSection = getFirstUncommittedSection();
1371 log.removeUntil(firstUncommittedSection);
1372 } else {
1373 setReadOnly = true;
1374 }
1375 }
1376 PageDataIndex systemTable = (PageDataIndex) metaObjects.get(0);
1377 isNew = systemTable == null;
1378 for (PageIndex index : metaObjects.values()) {
1379 if (index.getTable().isTemporary()) {
1380 // temporary indexes are removed after opening
1381 if (tempObjects == null) {
1382 tempObjects = New.hashMap();
1383 }
1384 tempObjects.put(index.getId(), index);
1385 } else {
1386 index.close(systemSession);
1387 }
1388 }
1389
1390 allocatePage(PAGE_ID_META_ROOT);
1391 writeIndexRowCounts();
1392 recoveryRunning = false;
1393 reservedPages = null;
1394
1395 writeBack();
1396 // clear the cache because it contains pages with closed indexes
1397 cache.clear();
1398 freeLists.clear();
1399
1400 metaObjects.clear();
1401 metaObjects.put(-1, metaIndex);
1402
1403 if (setReadOnly) {
1404 database.setReadOnly(true);
1405 }
1406 trace.debug("log recover done");
1407 return isEmpty;
1408 }
1409
1410 /**
1411 * A record is added to a table, or removed from a table.
1412 *
1413 * @param session the session
1414 * @param tableId the table id
1415 * @param row the row to add
1416 * @param add true if the row is added, false if it is removed
1417 */
1418 public synchronized void logAddOrRemoveRow(Session session, int tableId, Row row, boolean add) {
1419 if (logMode != LOG_MODE_OFF) {
1420 if (!recoveryRunning) {
1421 log.logAddOrRemoveRow(session, tableId, row, add);
1422 }
1423 }
1424 }
1425
1426 /**
1427 * Mark a committed transaction.
1428 *
1429 * @param session the session
1430 */
1431 public synchronized void commit(Session session) {
1432 checkOpen();
1433 openForWriting();
1434 log.commit(session.getId());
1435 long size = log.getSize();
1436 if (size - logSizeBase > maxLogSize) {
1437 int firstSection = log.getLogFirstSectionId();
1438 checkpoint();
1439 if (ignoreBigLog) {
1440 return;
1441 }
1442 int newSection = log.getLogSectionId();
1443 if (newSection - firstSection <= 2) {
1444 // one section is always kept, and checkpoint
1445 // advances two sections each time it is called
1446 return;
1447 }
1448 long newSize = log.getSize();
1449 if (newSize < size || size < maxLogSize) {
1450 ignoreBigLog = false;
1451 return;
1452 }
1453 ignoreBigLog = true;
1454 trace.error(null, "Transaction log could not be truncated; size: " + (newSize / 1024 / 1024) + " MB");
1455 logSizeBase = log.getSize();
1456 }
1457 }
1458
1459 /**
1460 * Prepare a transaction.
1461 *
1462 * @param session the session
1463 * @param transaction the name of the transaction
1464 */
1465 public synchronized void prepareCommit(Session session, String transaction) {
1466 log.prepareCommit(session, transaction);
1467 }
1468
1469 /**
1470 * Check whether this is a new database.
1471 *
1472 * @return true if it is
1473 */
1474 public boolean isNew() {
1475 return isNew;
1476 }
1477
1478 /**
1479 * Reserve the page if this is a index root page entry.
1480 *
1481 * @param logPos the redo log position
1482 * @param tableId the table id
1483 * @param row the row
1484 */
1485 void allocateIfIndexRoot(int logPos, int tableId, Row row) {
1486 if (tableId == META_TABLE_ID) {
1487 int rootPageId = row.getValue(3).getInt();
1488 if (reservedPages == null) {
1489 reservedPages = New.hashMap();
1490 }
1491 reservedPages.put(rootPageId, logPos);
1492 }
1493 }
1494
1495 /**
1496 * Redo a delete in a table.
1497 *
1498 * @param tableId the object id of the table
1499 * @param key the key of the row to delete
1500 */
1501 void redoDelete(int tableId, long key) {
1502 Index index = metaObjects.get(tableId);
1503 PageDataIndex scan = (PageDataIndex) index;
1504 Row row = scan.getRowWithKey(key);
1505 redo(tableId, row, false);
1506 }
1507
1508 /**
1509 * Redo a change in a table.
1510 *
1511 * @param tableId the object id of the table
1512 * @param row the row
1513 * @param add true if the record is added, false if deleted
1514 */
1515 void redo(int tableId, Row row, boolean add) {
1516 if (tableId == META_TABLE_ID) {
1517 if (add) {
1518 addMeta(row, systemSession, true);
1519 } else {
1520 removeMeta(row);
1521 }
1522 }
1523 Index index = metaObjects.get(tableId);
1524 if (index == null) {
1525 throw DbException.throwInternalError("Table not found: " + tableId + " " + row + " " + add);
1526 }
1527 Table table = index.getTable();
1528 if (add) {
1529 table.addRow(systemSession, row);
1530 } else {
1531 table.removeRow(systemSession, row);
1532 }
1533 }
1534
1535 /**
1536 * Redo a truncate.
1537 *
1538 * @param tableId the object id of the table
1539 */
1540 void redoTruncate(int tableId) {
1541 Index index = metaObjects.get(tableId);
1542 Table table = index.getTable();
1543 table.truncate(systemSession);
1544 }
1545
1546 private void openMetaIndex() {
1547 CreateTableData data = new CreateTableData();
1548 ArrayList<Column> cols = data.columns;
1549 cols.add(new Column("ID", Value.INT));
1550 cols.add(new Column("TYPE", Value.INT));
1551 cols.add(new Column("PARENT", Value.INT));
1552 cols.add(new Column("HEAD", Value.INT));
1553 cols.add(new Column("OPTIONS", Value.STRING));
1554 cols.add(new Column("COLUMNS", Value.STRING));
1555 metaSchema = new Schema(database, 0, "", null, true);
1556 data.schema = metaSchema;
1557 data.tableName = "PAGE_INDEX";
1558 data.id = META_TABLE_ID;
1559 data.temporary = false;
1560 data.persistData = true;
1561 data.persistIndexes = true;
1562 data.create = false;
1563 data.session = systemSession;
1564 metaTable = new RegularTable(data);
1565 metaIndex = (PageDataIndex) metaTable.getScanIndex(
1566 systemSession);
1567 metaObjects.clear();
1568 metaObjects.put(-1, metaIndex);
1569 }
1570
1571 private void readMetaData() {
1572 Cursor cursor = metaIndex.find(systemSession, null, null);
1573 // first, create all tables
1574 while (cursor.next()) {
1575 Row row = cursor.get();
1576 int type = row.getValue(1).getInt();
1577 if (type == META_TYPE_DATA_INDEX) {
1578 addMeta(row, systemSession, false);
1579 }
1580 }
1581 // now create all secondary indexes
1582 // otherwise the table might not be created yet
1583 cursor = metaIndex.find(systemSession, null, null);
1584 while (cursor.next()) {
1585 Row row = cursor.get();
1586 int type = row.getValue(1).getInt();
1587 if (type != META_TYPE_DATA_INDEX) {
1588 addMeta(row, systemSession, false);
1589 }
1590 }
1591 }
1592
1593 private void removeMeta(Row row) {
1594 int id = row.getValue(0).getInt();
1595 PageIndex index = metaObjects.get(id);
1596 index.getTable().removeIndex(index);
1597 if (index instanceof PageBtreeIndex || index instanceof PageDelegateIndex) {
1598 if (index.isTemporary()) {
1599 systemSession.removeLocalTempTableIndex(index);
1600 } else {
1601 index.getSchema().remove(index);
1602 }
1603 }
1604 index.remove(systemSession);
1605 metaObjects.remove(id);
1606 }
1607
1608 private void addMeta(Row row, Session session, boolean redo) {
1609 int id = row.getValue(0).getInt();
1610 int type = row.getValue(1).getInt();
1611 int parent = row.getValue(2).getInt();
1612 int rootPageId = row.getValue(3).getInt();
1613 String options = row.getValue(4).getString();
1614 String columnList = row.getValue(5).getString();
1615 String[] columns = StringUtils.arraySplit(columnList, ',', false);
1616 String[] ops = StringUtils.arraySplit(options, ',', false);
1617 Index meta;
1618 if (trace.isDebugEnabled()) {
1619 trace.debug("addMeta id="+ id +" type=" + type +
1620 " root=" + rootPageId + " parent=" + parent + " columns=" + columnList);
1621 }
1622 if (redo && rootPageId != 0) {
1623 // ensure the page is empty, but not used by regular data
1624 writePage(rootPageId, createData());
1625 allocatePage(rootPageId);
1626 }
1627 metaRootPageId.put(id, rootPageId);
1628 if (type == META_TYPE_DATA_INDEX) {
1629 CreateTableData data = new CreateTableData();
1630 if (SysProperties.CHECK) {
1631 if (columns == null) {
1632 throw DbException.throwInternalError(row.toString());
1633 }
1634 }
1635 for (int i = 0, len = columns.length; i < len; i++) {
1636 Column col = new Column("C" + i, Value.INT);
1637 data.columns.add(col);
1638 }
1639 data.schema = metaSchema;
1640 data.tableName = "T" + id;
1641 data.id = id;
1642 data.temporary = ops[2].equals("temp");
1643 data.persistData = true;
1644 data.persistIndexes = true;
1645 data.create = false;
1646 data.session = session;
1647 RegularTable table = new RegularTable(data);
1648 CompareMode mode = CompareMode.getInstance(ops[0], Integer.parseInt(ops[1]));
1649 table.setCompareMode(mode);
1650 meta = table.getScanIndex(session);
1651 } else {
1652 Index p = metaObjects.get(parent);
1653 if (p == null) {
1654 throw DbException.get(ErrorCode.FILE_CORRUPTED_1, "Table not found:" + parent + " for " + row + " meta:" + metaObjects);
1655 }
1656 RegularTable table = (RegularTable) p.getTable();
1657 Column[] tableCols = table.getColumns();
1658 int len = columns.length;
1659 IndexColumn[] cols = new IndexColumn[len];
1660 for (int i = 0; i < len; i++) {
1661 String c = columns[i];
1662 IndexColumn ic = new IndexColumn();
1663 int idx = c.indexOf('/');
1664 if (idx >= 0) {
1665 String s = c.substring(idx + 1);
1666 ic.sortType = Integer.parseInt(s);
1667 c = c.substring(0, idx);
1668 }
1669 Column column = tableCols[Integer.parseInt(c)];
1670 ic.column = column;
1671 cols[i] = ic;
1672 }
1673 IndexType indexType;
1674 if (ops[3].equals("d")) {
1675 indexType = IndexType.createPrimaryKey(true, false);
1676 Column[] tableColumns = table.getColumns();
1677 for (IndexColumn indexColumn : cols) {
1678 tableColumns[indexColumn.column.getColumnId()].setNullable(false);
1679 }
1680 } else {
1681 indexType = IndexType.createNonUnique(true);
1682 }
1683 meta = table.addIndex(session, "I" + id, id, cols, indexType, false, null);
1684 }
1685 PageIndex index;
1686 if (meta instanceof MultiVersionIndex) {
1687 index = (PageIndex) ((MultiVersionIndex) meta).getBaseIndex();
1688 } else {
1689 index = (PageIndex) meta;
1690 }
1691 metaObjects.put(id, index);
1692 }
1693
1694 /**
1695 * Add an index to the in-memory index map.
1696 *
1697 * @param index the index
1698 */
1699 public synchronized void addIndex(PageIndex index) {
1700 metaObjects.put(index.getId(), index);
1701 }
1702
1703 /**
1704 * Add the meta data of an index.
1705 *
1706 * @param index the index to add
1707 * @param session the session
1708 */
1709 public void addMeta(PageIndex index, Session session) {
1710 Table table = index.getTable();
1711 if (SysProperties.CHECK) {
1712 if (!table.isTemporary()) {
1713 // to prevent ABBA locking problems, we need to always take
1714 // the Database lock before we take the PageStore lock
1715 synchronized (database) {
1716 synchronized (this) {
1717 database.verifyMetaLocked(session);
1718 }
1719 }
1720 }
1721 }
1722 synchronized (this) {
1723 int type = index instanceof PageDataIndex ? META_TYPE_DATA_INDEX : META_TYPE_BTREE_INDEX;
1724 IndexColumn[] columns = index.getIndexColumns();
1725 StatementBuilder buff = new StatementBuilder();
1726 for (IndexColumn col : columns) {
1727 buff.appendExceptFirst(",");
1728 int id = col.column.getColumnId();
1729 buff.append(id);
1730 int sortType = col.sortType;
1731 if (sortType != 0) {
1732 buff.append('/');
1733 buff.append(sortType);
1734 }
1735 }
1736 String columnList = buff.toString();
1737 CompareMode mode = table.getCompareMode();
1738 String options = mode.getName()+ "," + mode.getStrength() + ",";
1739 if (table.isTemporary()) {
1740 options += "temp";
1741 }
1742 options += ",";
1743 if (index instanceof PageDelegateIndex) {
1744 options += "d";
1745 }
1746 Row row = metaTable.getTemplateRow();
1747 row.setValue(0, ValueInt.get(index.getId()));
1748 row.setValue(1, ValueInt.get(type));
1749 row.setValue(2, ValueInt.get(table.getId()));
1750 row.setValue(3, ValueInt.get(index.getRootPageId()));
1751 row.setValue(4, ValueString.get(options));
1752 row.setValue(5, ValueString.get(columnList));
1753 row.setKey(index.getId() + 1);
1754 metaIndex.add(session, row);
1755 }
1756 }
1757
1758 /**
1759 * Remove the meta data of an index.
1760 *
1761 * @param index the index to remove
1762 * @param session the session
1763 */
1764 public void removeMeta(Index index, Session session) {
1765 if (SysProperties.CHECK) {
1766 if (!index.getTable().isTemporary()) {
1767 // to prevent ABBA locking problems, we need to always take
1768 // the Database lock before we take the PageStore lock
1769 synchronized (database) {
1770 synchronized (this) {
1771 database.verifyMetaLocked(session);
1772 }
1773 }
1774 }
1775 }
1776 synchronized (this) {
1777 if (!recoveryRunning) {
1778 removeMetaIndex(index, session);
1779 metaObjects.remove(index.getId());
1780 }
1781 }
1782 }
1783
1784 private void removeMetaIndex(Index index, Session session) {
1785 int key = index.getId() + 1;
1786 Row row = metaIndex.getRow(session, key);
1787 if (row.getKey() != key) {
1788 throw DbException.get(ErrorCode.FILE_CORRUPTED_1,
1789 "key: " + key + " index: " + index +
1790 " table: " + index.getTable() + " row: " + row);
1791 }
1792 metaIndex.remove(session, row);
1793 }
1794
1795 /**
1796 * Set the maximum transaction log size in megabytes.
1797 *
1798 * @param maxSize the new maximum log size
1799 */
1800 public void setMaxLogSize(long maxSize) {
1801 this.maxLogSize = maxSize;
1802 }
1803
1804 /**
1805 * Commit or rollback a prepared transaction after opening a database with
1806 * in-doubt transactions.
1807 *
1808 * @param sessionId the session id
1809 * @param pageId the page where the transaction was prepared
1810 * @param commit if the transaction should be committed
1811 */
1812 public synchronized void setInDoubtTransactionState(int sessionId, int pageId, boolean commit) {
1813 boolean old = database.isReadOnly();
1814 try {
1815 database.setReadOnly(false);
1816 log.setInDoubtTransactionState(sessionId, pageId, commit);
1817 } finally {
1818 database.setReadOnly(old);
1819 }
1820 }
1821
1822 /**
1823 * Get the list of in-doubt transaction.
1824 *
1825 * @return the list
1826 */
1827 public ArrayList<InDoubtTransaction> getInDoubtTransactions() {
1828 return log.getInDoubtTransactions();
1829 }
1830
1831 /**
1832 * Check whether the recovery process is currently running.
1833 *
1834 * @return true if it is
1835 */
1836 public boolean isRecoveryRunning() {
1837 return recoveryRunning;
1838 }
1839
1840 private void checkOpen() {
1841 if (file == null) {
1842 throw DbException.get(ErrorCode.DATABASE_IS_CLOSED);
1843 }
1844 }
1845
1846 /**
1847 * Get the file write count since the database was created.
1848 *
1849 * @return the write count
1850 */
1851 public long getWriteCountTotal() {
1852 return writeCount + writeCountBase;
1853 }
1854
1855 /**
1856 * Get the file write count since the database was opened.
1857 *
1858 * @return the write count
1859 */
1860 public long getWriteCount() {
1861 return writeCount;
1862 }
1863
1864 /**
1865 * Get the file read count since the database was opened.
1866 *
1867 * @return the read count
1868 */
1869 public long getReadCount() {
1870 return readCount;
1871 }
1872
1873 /**
1874 * A table is truncated.
1875 *
1876 * @param session the session
1877 * @param tableId the table id
1878 */
1879 public synchronized void logTruncate(Session session, int tableId) {
1880 if (!recoveryRunning) {
1881 openForWriting();
1882 log.logTruncate(session, tableId);
1883 }
1884 }
1885
1886 /**
1887 * Get the root page of an index.
1888 *
1889 * @param indexId the index id
1890 * @return the root page
1891 */
1892 public int getRootPageId(int indexId) {
1893 return metaRootPageId.get(indexId);
1894 }
1895
1896 public Cache getCache() {
1897 return cache;
1898 }
1899
1900 private void checksumSet(byte[] d, int pageId) {
1901 int ps = pageSize;
1902 int type = d[0];
1903 if (type == Page.TYPE_EMPTY) {
1904 return;
1905 }
1906 int s1 = 255 + (type & 255), s2 = 255 + s1;
1907 s2 += s1 += d[6] & 255;
1908 s2 += s1 += d[(ps >> 1) - 1] & 255;
1909 s2 += s1 += d[ps >> 1] & 255;
1910 s2 += s1 += d[ps - 2] & 255;
1911 s2 += s1 += d[ps - 1] & 255;
1912 d[1] = (byte) (((s1 & 255) + (s1 >> 8)) ^ pageId);
1913 d[2] = (byte) (((s2 & 255) + (s2 >> 8)) ^ (pageId >> 8));
1914 }
1915
1916 /**
1917 * Check if the stored checksum is correct
1918 * @param d the data
1919 * @param pageId the page id
1920 * @param pageSize the page size
1921 * @return true if it is correct
1922 */
1923 public static boolean checksumTest(byte[] d, int pageId, int pageSize) {
1924 int ps = pageSize;
1925 int s1 = 255 + (d[0] & 255), s2 = 255 + s1;
1926 s2 += s1 += d[6] & 255;
1927 s2 += s1 += d[(ps >> 1) - 1] & 255;
1928 s2 += s1 += d[ps >> 1] & 255;
1929 s2 += s1 += d[ps - 2] & 255;
1930 s2 += s1 += d[ps - 1] & 255;
1931 if (d[1] != (byte) (((s1 & 255) + (s1 >> 8)) ^ pageId)
1932 || d[2] != (byte) (((s2 & 255) + (s2 >> 8)) ^ (pageId >> 8))) {
1933 return false;
1934 }
1935 return true;
1936 }
1937
1938 /**
1939 * Increment the change count. To be done after the operation has finished.
1940 */
1941 public void incrementChangeCount() {
1942 changeCount++;
1943 }
1944
1945 /**
1946 * Get the current change count. The first value is 1
1947 *
1948 * @return the change count
1949 */
1950 public int getChangeCount() {
1951 return changeCount;
1952 }
1953
1954 public void setLogMode(int logMode) {
1955 this.logMode = logMode;
1956 }
1957
1958 public int getLogMode() {
1959 return logMode;
1960 }
1961
1962 public void setLockFile(boolean lockFile) {
1963 this.lockFile = lockFile;
1964 }
1965
1966 public BitField getObjectIds() {
1967 BitField f = new BitField();
1968 Cursor cursor = metaIndex.find(systemSession, null, null);
1969 while (cursor.next()) {
1970 Row row = cursor.get();
1971 int id = row.getValue(0).getInt();
1972 if (id > 0) {
1973 f.set(id);
1974 }
1975 }
1976 return f;
1977 }
1978
1979 public Session getSystemSession() {
1980 return systemSession;
1981 }
1982
1983 public synchronized void setBackup(boolean start) {
1984 backupLevel += start ? 1 : -1;
1985 }
1986
1987 }
1988