Clover coverage report -
Coverage timestamp: Sun Nov 1 2009 23:08:24 UTC
file stats: LOC: 1,437   Methods: 99
NCLOC: 815   Classes: 5
 
 Source file Conditionals Statements Methods TOTAL
Paged.java 72.7% 81.8% 79.8% 79.9%
coverage coverage
 1    /*
 2    * Licensed to the Apache Software Foundation (ASF) under one or more
 3    * contributor license agreements. See the NOTICE file distributed with
 4    * this work for additional information regarding copyright ownership.
 5    * The ASF licenses this file to You under the Apache License, Version 2.0
 6    * (the "License"); you may not use this file except in compliance with
 7    * the License. You may obtain a copy of the License at
 8    *
 9    * http://www.apache.org/licenses/LICENSE-2.0
 10    *
 11    * Unless required by applicable law or agreed to in writing, software
 12    * distributed under the License is distributed on an "AS IS" BASIS,
 13    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14    * See the License for the specific language governing permissions and
 15    * limitations under the License.
 16    *
 17    * $Id: Paged.java 712683 2008-11-10 15:54:38Z natalia $
 18    */
 19   
 20    package org.apache.xindice.core.filer;
 21   
 22    import org.apache.commons.logging.Log;
 23    import org.apache.commons.logging.LogFactory;
 24    import org.apache.xindice.core.DBException;
 25    import org.apache.xindice.core.DBObject;
 26    import org.apache.xindice.core.FaultCodes;
 27    import org.apache.xindice.core.data.Key;
 28    import org.apache.xindice.core.data.Value;
 29    import org.apache.xindice.util.Configurable;
 30    import org.apache.xindice.util.Configuration;
 31   
 32    import java.io.ByteArrayInputStream;
 33    import java.io.ByteArrayOutputStream;
 34    import java.io.DataInput;
 35    import java.io.DataInputStream;
 36    import java.io.DataOutput;
 37    import java.io.DataOutputStream;
 38    import java.io.File;
 39    import java.io.IOException;
 40    import java.io.InputStream;
 41    import java.io.OutputStream;
 42    import java.io.RandomAccessFile;
 43    import java.lang.ref.WeakReference;
 44    import java.util.Collection;
 45    import java.util.EmptyStackException;
 46    import java.util.HashMap;
 47    import java.util.Iterator;
 48    import java.util.Map;
 49    import java.util.Stack;
 50    import java.util.WeakHashMap;
 51   
 52    /**
 53    * Paged is a paged file implementation that is foundation for both the
 54    * BTree class and the HashFiler. It provides flexible paged I/O and
 55    * page caching functionality.
 56    *
 57    * <br>
 58    * Paged has folowing configuration attributes:
 59    * <ul>
 60    * <li><strong>pagesize</strong>: Size of the page used by the paged file.
 61    * Default page size is 4096 bytes. This parameter can be set only
 62    * before paged file is created. Once it is created, this parameter
 63    * can not be changed.</li>
 64    * <li><strong>pagecount</strong>: Number of pages filer will be created
 65    * with.</li>
 66    * <li><strong>maxkeysize</strong>: Maximum allowed size of the key.
 67    * Default maximum key size is 256 bytes.</li>
 68    * <li><strong>max-descriptors</strong>: Defines maximum amount of
 69    * simultaneously opened file descriptors this paged file can have.
 70    * Several descriptors are needed to provide multithreaded access
 71    * to the underlying file. Too large number will limit amount of
 72    * collections you can open. Default value is 16
 73    * (DEFAULT_DESCRIPTORS_MAX).</li>
 74    * </ul>
 75    *
 76    * <br>FIXME: Currently it seems that maxkeysize is not used anywhere.
 77    * <br>TODO: Introduce Paged interface, implementations.
 78    *
 79    * @version $Revision: 712683 $, $Date: 2008-11-10 15:54:38 +0000 (Mon, 10 Nov 2008) $
 80    */
 81    public abstract class Paged implements DBObject, Configurable {
 82   
 83    private static final Log log = LogFactory.getLog(Paged.class);
 84   
 85    /**
 86    * The maximum number of pages that will be held in the dirty cache.
 87    * Once number reaches the limit, pages are flushed to disk.
 88    */
 89    private static final int MAX_DIRTY_SIZE = 128;
 90   
 91    /**
 92    * Name of the configuration attribute "pagesize"
 93    */
 94    protected static final String CONFIG_PAGESIZE = "pagesize";
 95   
 96    /**
 97    * Name of the configuration attribute "pagecount"
 98    */
 99    protected static final String CONFIG_PAGECOUNT = "pagecount";
 100   
 101    /**
 102    * Name of the configuration attribute "maxkeysize"
 103    */
 104    protected static final String CONFIG_KEYSIZE_MAX = "maxkeysize";
 105   
 106    /**
 107    * Name of the configuration attribute "max-descriptors"
 108    */
 109    protected static final String CONFIG_DESCRIPTORS_MAX = "max-descriptors";
 110   
 111    /**
 112    * Default value of the "pagesize".
 113    */
 114    private static final int DEFAULT_PAGESIZE = 4096;
 115   
 116    /**
 117    * Default value of the "pagecount".
 118    */
 119    private static final int DEFAULT_PAGECOUNT = 1024;
 120   
 121    /**
 122    * File header size
 123    */
 124    private static final int FILE_HEADER_SIZE = 4096;
 125   
 126    /**
 127    * Default value of the "maxkeysize".
 128    */
 129    private static final int DEFAULT_KEYSIZE_MAX = 256;
 130   
 131    /**
 132    * Default value of the maximum number of open random access files paged
 133    * can have. This number balances resources utilization and parallelism of
 134    * access to the paged file.
 135    */
 136    private static final int DEFAULT_DESCRIPTORS_MAX = 16;
 137   
 138   
 139    /**
 140    * Unused page status
 141    */
 142    protected static final byte UNUSED = 0;
 143   
 144    /**
 145    * Overflow page status
 146    */
 147    protected static final byte OVERFLOW = 126;
 148   
 149    /**
 150    * Deleted page status
 151    */
 152    protected static final byte DELETED = 127;
 153   
 154    /**
 155    * Page ID of non-existent page
 156    */
 157    protected static final int NO_PAGE = -1;
 158   
 159   
 160    /**
 161    * Configuration of this paged instance
 162    */
 163    private Configuration config;
 164   
 165    /**
 166    * Map of pages in use. Guarantees that page with same number will be loaded
 167    * into memory just once, allowing to synchronize on page objects to guarantee
 168    * no two threads are writing into same page at once.
 169    *
 170    * <p>Map contains weak references to the Page objects, keys are pages themselves.
 171    * Access is synchronized by the {@link #pagesLock}.
 172    */
 173    private final Map pages = new WeakHashMap();
 174   
 175    /**
 176    * Lock for synchronizing access to the {@link #pages} map.
 177    */
 178    private final Object pagesLock = new Object();
 179   
 180    /**
 181    * Cache of modified pages waiting to be written out.
 182    * Access is synchronized by the {@link #dirtyLock}.
 183    */
 184    private Map dirty = new HashMap();
 185   
 186    /**
 187    * Lock for synchronizing access to the {@link #dirty} map.
 188    */
 189    private final Object dirtyLock = new Object();
 190   
 191    /**
 192    * Random access file descriptors cache.
 193    * Access to it and to {@link #descriptorsCount} is synchronized by itself.
 194    */
 195    private final Stack descriptors = new Stack();
 196   
 197    /**
 198    * The number of random access file objects that exist, either in the
 199    * cache {@link #descriptors}, or currently in use.
 200    */
 201    private int descriptorsCount;
 202   
 203    /**
 204    * The maximum number of random access file objects that can be opened
 205    * by this paged instance.
 206    */
 207    private int descriptorsMax;
 208   
 209    /**
 210    * Whether the file is opened or not.
 211    */
 212    private boolean opened;
 213   
 214    /**
 215    * The underlying file where the Paged object stores its pages.
 216    */
 217    private File file;
 218   
 219    /**
 220    * Header of this Paged
 221    */
 222    private final FileHeader fileHeader;
 223   
 224   
 225  1411 public Paged() {
 226  1411 descriptorsMax = DEFAULT_DESCRIPTORS_MAX;
 227  1411 fileHeader = createFileHeader();
 228    }
 229   
 230  0 public Paged(File file) {
 231  0 this();
 232  0 setFile(file);
 233    }
 234   
 235  1411 public void setConfig(Configuration config) {
 236  1411 this.config = config;
 237    // Read paged config
 238  1411 fileHeader.setPageSize(config.getIntAttribute(CONFIG_PAGESIZE,
 239    fileHeader.getPageSize()));
 240  1411 fileHeader.setPageCount(config.getLongAttribute(CONFIG_PAGECOUNT,
 241    fileHeader.getPageCount()));
 242  1411 fileHeader.setMaxKeySize(config.getShortAttribute(CONFIG_KEYSIZE_MAX,
 243    fileHeader.getMaxKeySize()));
 244  1411 descriptorsMax = getConfig().getIntAttribute(CONFIG_DESCRIPTORS_MAX,
 245    descriptorsMax);
 246    }
 247   
 248  1411 public Configuration getConfig() {
 249  1411 return config;
 250    }
 251   
 252    /**
 253    * setFile sets the file object for this Paged.
 254    *
 255    * @param file The File
 256    */
 257  1411 protected final void setFile(final File file) {
 258  1411 this.file = file;
 259    }
 260   
 261    /**
 262    * getFile returns the file object for this Paged.
 263    *
 264    * @return The File
 265    */
 266  1123 protected final File getFile() {
 267  1123 return file;
 268    }
 269   
 270    /**
 271    * Obtain RandomAccessFile ('descriptor') object out of the pool.
 272    * If no descriptors available, and maximum amount already allocated,
 273    * the call will block.
 274    */
 275  104911 protected final RandomAccessFile getDescriptor() throws IOException {
 276  104911 synchronized (descriptors) {
 277    // If there are descriptors in the cache return one.
 278  104911 if (!descriptors.empty()) {
 279  99781 return (RandomAccessFile) descriptors.pop();
 280    }
 281    // Otherwise we need to get one some other way.
 282   
 283    // First try to create a new one if there's room
 284  5130 if (descriptorsCount < descriptorsMax) {
 285  5130 descriptorsCount++;
 286  5130 return new RandomAccessFile(file, "rw");
 287    }
 288   
 289    // Otherwise we have to wait for one to be released by another thread.
 290  0 while (true) {
 291  0 try {
 292  0 descriptors.wait();
 293  0 return (RandomAccessFile) descriptors.pop();
 294    } catch (InterruptedException e) {
 295    // Ignore, and continue to wait
 296    } catch (EmptyStackException e) {
 297    // Ignore, and continue to wait
 298    }
 299    }
 300    }
 301    }
 302   
 303    /**
 304    * Puts a RandomAccessFile ('descriptor') back into the descriptor pool.
 305    */
 306  104910 protected final void putDescriptor(RandomAccessFile raf) {
 307  104910 if (raf != null) {
 308  104909 synchronized (descriptors) {
 309  104911 descriptors.push(raf);
 310  104911 descriptors.notify();
 311    }
 312    }
 313    }
 314   
 315    /**
 316    * Closes a RandomAccessFile ('descriptor') and removes it from the pool.
 317    */
 318  5130 protected final void closeDescriptor(RandomAccessFile raf) {
 319  5130 if (raf != null) {
 320  5130 try {
 321  5130 raf.close();
 322    } catch (IOException e) {
 323    // Ignore close exception
 324    }
 325   
 326    // Synchronization is necessary as decrement operation is not atomic
 327  5130 synchronized (descriptors) {
 328  5130 descriptorsCount --;
 329    }
 330    }
 331    }
 332   
 333    /**
 334    * getPage returns the page specified by pageNum.
 335    *
 336    * @param pageNum The Page number
 337    * @return The requested Page
 338    * @throws IOException if an Exception occurs
 339    */
 340  118455 protected final Page getPage(long pageNum) throws IOException {
 341  118455 Page p = updateIdentityMap(pageNum);
 342   
 343    // Load the page from disk if necessary
 344  118457 p.read();
 345  118456 return p;
 346    }
 347   
 348  130621 private Page updateIdentityMap(long pageNum) {
 349  130622 final PageKey k = new PageKey(pageNum);
 350  130624 Page p = null;
 351  130625 synchronized (pagesLock) {
 352    // Check if page is already loaded in the page cache
 353  130626 WeakReference ref = (WeakReference) pages.get(k);
 354  130626 if (ref != null) {
 355  77430 p = (Page) ref.get();
 356    // Fall through to p.read(). Even if page present in the pages
 357    // map, it still has to be read - it could be that it was just
 358    // added to the map but read() was not called yet.
 359    }
 360   
 361    // If not found, create it and add it to the pages cache
 362  130626 if (p == null) {
 363  53196 p = new Page(pageNum);
 364  53196 pages.put(p, new WeakReference(p));
 365    }
 366    }
 367   
 368  130626 return p;
 369    }
 370   
 371    /**
 372    * readValue reads the multi-Paged Value starting at the specified
 373    * Page.
 374    *
 375    * @param page The starting Page
 376    * @return The Value
 377    * @throws IOException if an Exception occurs
 378    */
 379  40657 protected final Value readValue(Page page) throws IOException {
 380  40657 final PageHeader sph = page.getPageHeader();
 381  40657 ByteArrayOutputStream bos = new ByteArrayOutputStream(sph.getRecordLen());
 382   
 383    // Loop until we've read all the pages into memory.
 384  40657 Page p = page;
 385  40657 while (true) {
 386  40658 PageHeader ph = p.getPageHeader();
 387   
 388    // Add the contents of the page onto the stream
 389  40658 p.streamTo(bos);
 390   
 391    // Continue following the list of pages until we get to the end.
 392  40658 long nextPage = ph.getNextPage();
 393  40658 if (nextPage == NO_PAGE) {
 394  40657 break;
 395    }
 396  1 p = getPage(nextPage);
 397    }
 398   
 399    // Return a Value with the collected contents of all pages.
 400  40657 return new Value(bos.toByteArray());
 401    }
 402   
 403    /**
 404    * readValue reads the multi-Paged Value starting at the specified
 405    * page number.
 406    *
 407    * @param page The starting page number
 408    * @return The Value
 409    * @throws IOException if an Exception occurs
 410    */
 411  0 protected final Value readValue(long page) throws IOException {
 412  0 return readValue(getPage(page));
 413    }
 414   
 415    /**
 416    * writeValue writes the multi-Paged Value starting at the specified
 417    * Page.
 418    *
 419    * @param page The starting Page
 420    * @param value The Value to write
 421    * @throws IOException if an Exception occurs
 422    */
 423  34316 protected final void writeValue(Page page, Value value) throws IOException {
 424  34316 if (value == null) {
 425  0 throw new IOException("Can't write a null value");
 426    }
 427   
 428  34316 InputStream is = value.getInputStream();
 429   
 430    // Write as much as we can onto the primary page.
 431  34316 PageHeader hdr = page.getPageHeader();
 432  34316 hdr.setRecordLen(value.getLength());
 433  34316 page.streamFrom(is);
 434   
 435    // Write out the rest of the value onto any needed overflow pages
 436  34316 while (is.available() > 0) {
 437  3 Page lpage = page;
 438  3 PageHeader lhdr = hdr;
 439   
 440    // Find an overflow page to use
 441  3 long np = lhdr.getNextPage();
 442  3 if (np != NO_PAGE) {
 443    // Use an existing page.
 444  0 page = getPage(np);
 445    } else {
 446    // Create a new overflow page
 447  3 page = getFreePage();
 448  3 lhdr.setNextPage(page.getPageNum());
 449    }
 450   
 451    // Mark the page as an overflow page.
 452  3 hdr = page.getPageHeader();
 453  3 hdr.setStatus(OVERFLOW);
 454   
 455    // Write some more of the value to the overflow page.
 456  3 page.streamFrom(is);
 457  3 lpage.write();
 458    }
 459   
 460    // Cleanup any unused overflow pages. i.e. the value is smaller then the
 461    // last time it was written.
 462  34316 long np = hdr.getNextPage();
 463  34316 if (np != NO_PAGE) {
 464  1 unlinkPages(np);
 465    }
 466   
 467  34316 hdr.setNextPage(NO_PAGE);
 468  34316 page.write();
 469    }
 470   
 471    /**
 472    * writeValue writes the multi-Paged Value starting at the specified
 473    * page number.
 474    *
 475    * @param page The starting page number
 476    * @param value The Value to write
 477    * @throws IOException if an Exception occurs
 478    */
 479  0 protected final void writeValue(long page, Value value) throws IOException {
 480  0 writeValue(getPage(page), value);
 481    }
 482   
 483    /**
 484    * unlinkPages unlinks a set of pages starting at the specified Page.
 485    *
 486    * @param page The starting Page to unlink
 487    * @throws IOException if an Exception occurs
 488    */
 489  3621 protected void unlinkPages(Page page) throws IOException {
 490    // Add any overflow pages to the list of free pages.
 491    // Get the first and last page in the chain.
 492  3621 long firstPage = page.pageNum;
 493  3621 while (page.header.nextPage != NO_PAGE) {
 494  2 page = getPage(page.header.nextPage);
 495    }
 496  3621 long lastPage = page.pageNum;
 497   
 498    // Free the chain
 499  3621 synchronized (fileHeader) {
 500    // If there are already some free pages, add the start of the chain
 501    // to the list of free pages.
 502  3621 if (fileHeader.lastFreePage != NO_PAGE) {
 503  3170 Page p = getPage(fileHeader.lastFreePage);
 504  3170 p.header.setNextPage(firstPage);
 505  3170 p.write();
 506    }
 507   
 508    // Otherwise set the chain as the list of free pages.
 509  3621 if (fileHeader.firstFreePage == NO_PAGE) {
 510  451 fileHeader.setFirstFreePage(firstPage);
 511    }
 512   
 513    // Add a reference to the end of the chain.
 514  3621 fileHeader.setLastFreePage(lastPage);
 515    }
 516    }
 517   
 518    /**
 519    * unlinkPages unlinks a set of pages starting at the specified
 520    * page number.
 521    *
 522    * @param pageNum The starting page number to unlink
 523    * @throws IOException if an Exception occurs
 524    */
 525  1 protected final void unlinkPages(long pageNum) throws IOException {
 526  1 unlinkPages(getPage(pageNum));
 527    }
 528   
 529    /**
 530    * getFreePage returns the first free Page from secondary storage.
 531    * If no Pages are available, the file is grown as appropriate.
 532    *
 533    * @return The next free Page
 534    * @throws IOException if an Exception occurs
 535    */
 536  13059 protected final Page getFreePage() throws IOException {
 537  13059 Page p = null;
 538   
 539    // Synchronize read and write to the fileHeader.firstFreePage
 540  13059 synchronized (fileHeader) {
 541  13059 if (fileHeader.firstFreePage != NO_PAGE) {
 542    // Steal a deleted page
 543  890 p = getPage(fileHeader.firstFreePage);
 544  890 fileHeader.setFirstFreePage(p.getPageHeader().nextPage);
 545  890 if (fileHeader.firstFreePage == NO_PAGE) {
 546  52 fileHeader.setLastFreePage(NO_PAGE);
 547    }
 548    }
 549    }
 550   
 551  13059 if (p == null) {
 552    // No deleted pages, grow the file
 553  12169 p = updateIdentityMap(fileHeader.incTotalCount());
 554   
 555    // initialize page
 556  12169 p.data = new byte[fileHeader.pageSize];
 557  12169 p.keyPos = fileHeader.pageHeaderSize;
 558  12169 p.dataPos = p.keyPos + p.header.keyLen;
 559    }
 560   
 561    // Initialize The Page Header (Cleanly)
 562  13059 p.header.setNextPage(NO_PAGE);
 563  13058 p.header.setStatus(UNUSED);
 564  13058 return p;
 565    }
 566   
 567    /**
 568    * @throws DBException COL_COLLECTION_CLOSED if paged file is closed
 569    */
 570  69047 protected final void checkOpened() throws DBException {
 571  69048 if (!opened) {
 572  0 throw new FilerException(FaultCodes.COL_COLLECTION_CLOSED,
 573    "Filer is closed");
 574    }
 575    }
 576   
 577    /**
 578    * getFileHeader returns the FileHeader
 579    *
 580    * @return The FileHeader
 581    */
 582  2714 protected FileHeader getFileHeader() {
 583  2714 return fileHeader;
 584    }
 585   
 586    /**
 587    * @return True if this paged file exists
 588    */
 589  5091 public boolean exists() {
 590  5091 return file.exists();
 591    }
 592   
 593  1170 public boolean create() throws DBException {
 594  1170 try {
 595  1170 createFile();
 596  1170 fileHeader.write();
 597  1170 flush();
 598  1170 return true;
 599    } catch (Exception e) {
 600  0 throw new FilerException(FaultCodes.GEN_CRITICAL_ERROR,
 601    "Error creating " + file.getName(), e);
 602    }
 603    }
 604   
 605  1170 private void createFile() throws IOException {
 606  1170 RandomAccessFile raf = null;
 607  1170 try {
 608  1170 raf = getDescriptor();
 609  1170 long o = fileHeader.headerSize + (fileHeader.pageCount + 1) * fileHeader.pageSize - 1;
 610  1170 raf.seek(o);
 611  1170 raf.write(0);
 612    } finally {
 613  1170 putDescriptor(raf);
 614    }
 615    }
 616   
 617  2559 public synchronized boolean open() throws DBException {
 618  2559 RandomAccessFile raf = null;
 619  2559 try {
 620  2559 if (exists()) {
 621  2559 raf = getDescriptor();
 622  2559 fileHeader.read();
 623   
 624    // This is the only property that can be changed after creation
 625  2559 fileHeader.setMaxKeySize(config.getShortAttribute(CONFIG_KEYSIZE_MAX,
 626    fileHeader.getMaxKeySize()));
 627   
 628  2559 opened = true;
 629    } else {
 630  0 opened = false;
 631    }
 632  2559 return opened;
 633    } catch (Exception e) {
 634  0 throw new FilerException(FaultCodes.GEN_CRITICAL_ERROR,
 635    "Error opening " + file.getName(), e);
 636    } finally {
 637  2559 putDescriptor(raf);
 638    }
 639    }
 640   
 641  2560 public synchronized boolean close() throws DBException {
 642  2560 if (isOpened()) {
 643  2559 try {
 644    // First of all, mark as closed to prevent operations
 645  2559 opened = false;
 646  2559 flush();
 647   
 648  2559 synchronized (descriptors) {
 649  2559 final int total = descriptorsCount;
 650    // Close descriptors in cache
 651  2559 while (!descriptors.empty()) {
 652  5130 closeDescriptor((RandomAccessFile)descriptors.pop());
 653    }
 654    // Attempt to close descriptors in use. Max wait time = 0.5s * MAX_DESCRIPTORS
 655  2559 int n = descriptorsCount;
 656  2559 while (descriptorsCount > 0 && n > 0) {
 657  0 descriptors.wait(500);
 658  0 if (descriptors.isEmpty()) {
 659  0 n--;
 660    } else {
 661  0 closeDescriptor((RandomAccessFile)descriptors.pop());
 662    }
 663    }
 664  2559 if (descriptorsCount > 0) {
 665  0 log.warn(descriptorsCount + " out of " + total + " files were not closed during close.");
 666    }
 667    }
 668   
 669    // clear cache
 670  2559 synchronized (pagesLock) {
 671  2559 pages.clear();
 672    }
 673    } catch (Exception e) {
 674    // Failed to close, leave open
 675  0 opened = true;
 676  0 throw new FilerException(FaultCodes.GEN_CRITICAL_ERROR,
 677    "Error closing " + file.getName(), e);
 678    }
 679    }
 680   
 681  2560 return true;
 682    }
 683   
 684  78916 public boolean isOpened() {
 685  78916 return opened;
 686    }
 687   
 688  1124 public boolean drop() throws DBException {
 689  1124 try {
 690  1124 close();
 691  1124 if (exists()) {
 692  1123 return getFile().delete();
 693    } else {
 694  1 return true;
 695    }
 696    } catch (Exception e) {
 697  0 throw new FilerException(FaultCodes.COL_CANNOT_DROP,
 698    "Can't drop " + file.getName(), e);
 699    }
 700    }
 701   
 702  41200 void addDirty(Page page) throws IOException {
 703  41200 boolean flush;
 704  41200 synchronized (dirtyLock) {
 705  41200 dirty.put(page, page);
 706  41200 flush = dirty.size() > MAX_DIRTY_SIZE;
 707    }
 708   
 709  41200 if (flush) {
 710    // Too many dirty pages... flush them
 711  6 try {
 712  6 flush();
 713    } catch (Exception e) {
 714  0 throw new IOException(e.getMessage());
 715    }
 716    }
 717    }
 718   
 719  24520 public void flush() throws DBException {
 720    // This method is not synchronized
 721   
 722    // Error flag/counter
 723  24520 int error = 0;
 724   
 725    // Obtain collection of dirty pages
 726  24520 Collection pages;
 727  24520 synchronized (dirtyLock) {
 728  24520 pages = dirty.values();
 729  24520 dirty = new HashMap();
 730    }
 731   
 732    // Flush dirty pages
 733  24520 Iterator i = pages.iterator();
 734  24520 while (i.hasNext()) {
 735  36456 Page p = (Page) i.next();
 736  36458 try {
 737  36458 p.flush();
 738    } catch (Exception e) {
 739  0 log.warn("Exception while flushing page " + p.pageNum, e);
 740  0 error++;
 741    }
 742    }
 743   
 744    // Flush header
 745  24520 if (fileHeader.dirty) {
 746  21536 try {
 747  21536 fileHeader.write();
 748    } catch (Exception e) {
 749  0 log.warn("Exception while flushing file header", e);
 750  0 error++;
 751    }
 752    }
 753   
 754  24520 if (error != 0) {
 755  0 throw new FilerException(FaultCodes.GEN_CRITICAL_ERROR,
 756    "Error performing flush! Failed to flush " + error + " pages!");
 757    }
 758    }
 759   
 760   
 761    /**
 762    * createFileHeader must be implemented by a Paged implementation
 763    * in order to create an appropriate subclass instance of a FileHeader.
 764    *
 765    * @return a new FileHeader
 766    */
 767    protected abstract FileHeader createFileHeader();
 768   
 769    /**
 770    * createPageHeader must be implemented by a Paged implementation
 771    * in order to create an appropriate subclass instance of a PageHeader.
 772    *
 773    * @return a new PageHeader
 774    */
 775    protected abstract PageHeader createPageHeader();
 776   
 777   
 778    // These are a bunch of utility methods for subclasses
 779   
 780  13164 public static Value[] insertArrayValue(Value[] vals, Value val, int idx) {
 781  13164 Value[] newVals = new Value[vals.length + 1];
 782  13164 if (idx > 0) {
 783  11151 System.arraycopy(vals, 0, newVals, 0, idx);
 784    }
 785  13164 newVals[idx] = val;
 786  13164 if (idx < vals.length) {
 787  5301 System.arraycopy(vals, idx, newVals, idx + 1, vals.length - idx);
 788    }
 789  13164 return newVals;
 790    }
 791   
 792  2745 public static Value[] deleteArrayValue(Value[] vals, int idx) {
 793  2745 Value[] newVals = new Value[vals.length - 1];
 794  2745 if (idx > 0) {
 795  1242 System.arraycopy(vals, 0, newVals, 0, idx);
 796    }
 797  2745 if (idx < newVals.length) {
 798  2044 System.arraycopy(vals, idx + 1, newVals, idx, newVals.length - idx);
 799    }
 800  2745 return newVals;
 801    }
 802   
 803  13164 public static long[] insertArrayLong(long[] vals, long val, int idx) {
 804  13164 long[] newVals = new long[vals.length + 1];
 805  13164 if (idx > 0) {
 806  11157 System.arraycopy(vals, 0, newVals, 0, idx);
 807    }
 808  13164 newVals[idx] = val;
 809  13164 if (idx < vals.length) {
 810  5301 System.arraycopy(vals, idx, newVals, idx + 1, vals.length - idx);
 811    }
 812  13164 return newVals;
 813    }
 814   
 815  2745 public static long[] deleteArrayLong(long[] vals, int idx) {
 816  2745 long[] newVals = new long[vals.length - 1];
 817  2745 if (idx > 0) {
 818  1242 System.arraycopy(vals, 0, newVals, 0, idx);
 819    }
 820  2745 if (idx < newVals.length) {
 821  2044 System.arraycopy(vals, idx + 1, newVals, idx, newVals.length - idx);
 822    }
 823  2745 return newVals;
 824    }
 825   
 826  0 public static int[] insertArrayInt(int[] vals, int val, int idx) {
 827  0 int[] newVals = new int[vals.length + 1];
 828  0 if (idx > 0) {
 829  0 System.arraycopy(vals, 0, newVals, 0, idx);
 830    }
 831  0 newVals[idx] = val;
 832  0 if (idx < vals.length) {
 833  0 System.arraycopy(vals, idx, newVals, idx + 1, vals.length - idx);
 834    }
 835  0 return newVals;
 836    }
 837   
 838  0 public static int[] deleteArrayInt(int[] vals, int idx) {
 839  0 int[] newVals = new int[vals.length - 1];
 840  0 if (idx > 0) {
 841  0 System.arraycopy(vals, 0, newVals, 0, idx);
 842    }
 843  0 if (idx < newVals.length) {
 844  0 System.arraycopy(vals, idx + 1, newVals, idx, newVals.length - idx);
 845    }
 846  0 return newVals;
 847    }
 848   
 849  0 public static short[] insertArrayShort(short[] vals, short val, int idx) {
 850  0 short[] newVals = new short[vals.length + 1];
 851  0 if (idx > 0) {
 852  0 System.arraycopy(vals, 0, newVals, 0, idx);
 853    }
 854  0 newVals[idx] = val;
 855  0 if (idx < vals.length) {
 856  0 System.arraycopy(vals, idx, newVals, idx + 1, vals.length - idx);
 857    }
 858   
 859  0 return newVals;
 860    }
 861   
 862  0 public static short[] deleteArrayShort(short[] vals, int idx) {
 863  0 short[] newVals = new short[vals.length - 1];
 864  0 if (idx > 0) {
 865  0 System.arraycopy(vals, 0, newVals, 0, idx);
 866    }
 867  0 if (idx < newVals.length) {
 868  0 System.arraycopy(vals, idx + 1, newVals, idx, newVals.length - idx);
 869    }
 870   
 871  0 return newVals;
 872    }
 873   
 874   
 875    /**
 876    * Paged file's header
 877    */
 878    protected abstract class FileHeader {
 879    private boolean dirty;
 880    private int workSize;
 881   
 882    private short headerSize;
 883   
 884    /**
 885    * Size of the page in bytes.
 886    */
 887    private int pageSize;
 888   
 889    /**
 890    * Number of pages initially allocated for the file.
 891    * Has a special (historical) meaning for HashFiler.
 892    */
 893    private long pageCount;
 894   
 895    /**
 896    * Number of pages used by the filer. Initially set to 0.
 897    * Has somewhat different (historical) meaning for HashFiler.
 898    */
 899    private long totalCount;
 900   
 901    private long firstFreePage = -1;
 902    private long lastFreePage = -1;
 903    private byte pageHeaderSize = 64;
 904    private short maxKeySize = DEFAULT_KEYSIZE_MAX;
 905    private long recordCount;
 906   
 907   
 908  1411 public FileHeader() {
 909  1411 this.pageSize = DEFAULT_PAGESIZE;
 910  1411 this.pageCount = DEFAULT_PAGECOUNT;
 911  1411 this.headerSize = (short) FILE_HEADER_SIZE;
 912  1411 calculateWorkSize();
 913    }
 914   
 915  2559 public synchronized final void read() throws IOException {
 916  2559 RandomAccessFile raf = getDescriptor();
 917  2559 try {
 918  2559 raf.seek(0);
 919  2559 read(raf);
 920  2559 calculateWorkSize();
 921    } finally {
 922  2559 putDescriptor(raf);
 923    }
 924    }
 925   
 926  2559 protected synchronized void read(RandomAccessFile raf) throws IOException {
 927  2559 headerSize = raf.readShort();
 928  2559 pageSize = raf.readInt();
 929  2559 pageCount = raf.readLong();
 930  2559 totalCount = raf.readLong();
 931  2559 firstFreePage = raf.readLong();
 932  2559 lastFreePage = raf.readLong();
 933  2559 pageHeaderSize = raf.readByte();
 934  2559 maxKeySize = raf.readShort();
 935  2559 recordCount = raf.readLong();
 936    }
 937   
 938  22706 public synchronized final void write() throws IOException {
 939  22706 if (dirty) {
 940  21138 RandomAccessFile raf = getDescriptor();
 941  21138 try {
 942  21138 raf.seek(0);
 943  21138 write(raf);
 944  21138 dirty = false;
 945    } finally {
 946  21138 putDescriptor(raf);
 947    }
 948    }
 949    }
 950   
 951  21138 protected synchronized void write(RandomAccessFile raf) throws IOException {
 952  21138 raf.writeShort(headerSize);
 953  21138 raf.writeInt(pageSize);
 954  21138 raf.writeLong(pageCount);
 955  21138 raf.writeLong(totalCount);
 956  21138 raf.writeLong(firstFreePage);
 957  21138 raf.writeLong(lastFreePage);
 958  21138 raf.writeByte(pageHeaderSize);
 959  21138 raf.writeShort(maxKeySize);
 960  21138 raf.writeLong(recordCount);
 961    }
 962   
 963  26490 public synchronized final void setDirty() {
 964  26490 dirty = true;
 965    }
 966   
 967  0 public synchronized final boolean isDirty() {
 968  0 return dirty;
 969    }
 970   
 971    /**
 972    * The size of the FileHeader. Usually 1 OS Page.
 973    * This method should be called only while initializing Paged, not during normal processing.
 974    */
 975  0 public synchronized final void setHeaderSize(short headerSize) {
 976  0 this.headerSize = headerSize;
 977  0 dirty = true;
 978    }
 979   
 980    /** The size of the FileHeader. Usually 1 OS Page */
 981  0 public synchronized final short getHeaderSize() {
 982  0 return headerSize;
 983    }
 984   
 985    /**
 986    * The size of a page. Usually a multiple of a FS block.
 987    * This method should be called only while initializing Paged, not during normal processing.
 988    */
 989  1475 public synchronized final void setPageSize(int pageSize) {
 990  1475 this.pageSize = pageSize;
 991  1475 calculateWorkSize();
 992  1475 dirty = true;
 993    }
 994   
 995    /** The size of a page. Usually a multiple of a FS block */
 996  4429 public synchronized final int getPageSize() {
 997  4429 return pageSize;
 998    }
 999   
 1000    /**
 1001    * The number of pages in primary/initial storage.
 1002    * This method should be called only while initializing Paged, not during normal processing.
 1003    */
 1004  1411 public synchronized final void setPageCount(long pageCount) {
 1005  1411 this.pageCount = pageCount;
 1006  1411 dirty = true;
 1007    }
 1008   
 1009    /** The number of pages in primary storage */
 1010  11413 public synchronized final long getPageCount() {
 1011  11413 return pageCount;
 1012    }
 1013   
 1014    /**
 1015    * The number of used pages in the file.
 1016    * This method should be called only while initializing Paged, not during normal processing.
 1017    */
 1018  38 public synchronized final void setTotalCount(long totalCount) {
 1019  38 this.totalCount = totalCount;
 1020  38 dirty = true;
 1021    }
 1022   
 1023  12169 public synchronized final long incTotalCount() {
 1024  12169 dirty = true;
 1025  12169 return this.totalCount++;
 1026    }
 1027   
 1028    /** The number of used pages in the file */
 1029  2 public synchronized final long getTotalCount() {
 1030  2 return totalCount;
 1031    }
 1032   
 1033    /** The first free page in unused secondary space */
 1034  1341 public synchronized final void setFirstFreePage(long firstFreePage) {
 1035  1341 this.firstFreePage = firstFreePage;
 1036  1341 dirty = true;
 1037    }
 1038   
 1039    /** The first free page in unused secondary space */
 1040  0 public synchronized final long getFirstFreePage() {
 1041  0 return firstFreePage;
 1042    }
 1043   
 1044    /** The last free page in unused secondary space */
 1045  3673 public synchronized final void setLastFreePage(long lastFreePage) {
 1046  3673 this.lastFreePage = lastFreePage;
 1047  3673 dirty = true;
 1048    }
 1049   
 1050    /** The last free page in unused secondary space */
 1051  0 public synchronized final long getLastFreePage() {
 1052  0 return lastFreePage;
 1053    }
 1054   
 1055    /**
 1056    * Set the size of a page header.
 1057    *
 1058    * Normally, 64 is sufficient.
 1059    */
 1060  0 public synchronized final void setPageHeaderSize(byte pageHeaderSize) {
 1061  0 this.pageHeaderSize = pageHeaderSize;
 1062  0 calculateWorkSize();
 1063  0 dirty = true;
 1064    }
 1065   
 1066    /**
 1067    * Get the size of a page header.
 1068    */
 1069  44218 public synchronized final byte getPageHeaderSize() {
 1070  44218 return pageHeaderSize;
 1071    }
 1072   
 1073    /**
 1074    * Set the maximum number of bytes a key can be.
 1075    *
 1076    * Normally, 256 is good
 1077    */
 1078  3970 public synchronized final void setMaxKeySize(short maxKeySize) {
 1079  3970 this.maxKeySize = maxKeySize;
 1080  3970 dirty = true;
 1081    }
 1082   
 1083    /**
 1084    * Get the maximum number of bytes a key can be.
 1085    */
 1086  3970 public synchronized final short getMaxKeySize() {
 1087  3970 return maxKeySize;
 1088    }
 1089   
 1090    /** Increment the number of records being managed by the file */
 1091  11175 public synchronized final void incRecordCount() {
 1092  11175 recordCount++;
 1093  11175 dirty = true;
 1094    }
 1095   
 1096    /** Decrement the number of records being managed by the file */
 1097  3753 public synchronized final void decRecordCount() {
 1098  3753 recordCount--;
 1099  3753 dirty = true;
 1100    }
 1101   
 1102    /** The number of records being managed by the file (not pages) */
 1103  194 public synchronized final long getRecordCount() {
 1104  194 return recordCount;
 1105    }
 1106   
 1107  5445 private synchronized void calculateWorkSize() {
 1108  5445 workSize = pageSize - pageHeaderSize;
 1109    }
 1110   
 1111  27246 public synchronized final int getWorkSize() {
 1112  27246 return workSize;
 1113    }
 1114    }
 1115   
 1116    /**
 1117    * Paged file page's header
 1118    */
 1119    protected abstract static class PageHeader implements Streamable {
 1120    private boolean dirty;
 1121    private byte status = UNUSED;
 1122    private short keyLen;
 1123    private int keyHash;
 1124    private int dataLen;
 1125    private int recordLen;
 1126    private long nextPage = NO_PAGE;
 1127   
 1128  53196 public PageHeader() {
 1129    }
 1130   
 1131  0 public PageHeader(DataInput dis) throws IOException {
 1132  0 read(dis);
 1133    }
 1134   
 1135  41027 public synchronized void read(DataInput dis) throws IOException {
 1136  41027 status = dis.readByte();
 1137  41027 dirty = false;
 1138  41027 if (status == UNUSED) {
 1139  2559 return;
 1140    }
 1141   
 1142  38468 keyLen = dis.readShort();
 1143  38468 keyHash = dis.readInt();
 1144  38468 dataLen = dis.readInt();
 1145  38468 recordLen = dis.readInt();
 1146  38468 nextPage = dis.readLong();
 1147    }
 1148   
 1149  41200 public synchronized void write(DataOutput dos) throws IOException {
 1150  41200 dirty = false;
 1151  41200 dos.writeByte(status);
 1152  41200 dos.writeShort(keyLen);
 1153  41200 dos.writeInt(keyHash);
 1154  41200 dos.writeInt(dataLen);
 1155  41200 dos.writeInt(recordLen);
 1156  41200 dos.writeLong(nextPage);
 1157    }
 1158   
 1159  0 public synchronized final boolean isDirty() {
 1160  0 return dirty;
 1161    }
 1162   
 1163  52840 public synchronized final void setDirty() {
 1164  52840 dirty = true;
 1165    }
 1166   
 1167    /** The status of this page (UNUSED, RECORD, DELETED, etc...) */
 1168  31623 public synchronized final void setStatus(byte status) {
 1169  31622 this.status = status;
 1170  31624 dirty = true;
 1171    }
 1172   
 1173    /** The status of this page (UNUSED, RECORD, DELETED, etc...) */
 1174  307054 public synchronized final byte getStatus() {
 1175  307054 return status;
 1176    }
 1177   
 1178  3017 public synchronized final void setKey(Key key) {
 1179    // setKey WIPES OUT the Page data
 1180  3017 setRecordLen(0);
 1181  3015 dataLen = 0;
 1182  3016 keyHash = key.hashCode();
 1183  3017 keyLen = (short) key.getLength();
 1184  3017 dirty = true;
 1185    }
 1186   
 1187    /** The length of the Key */
 1188  0 public synchronized final void setKeyLen(short keyLen) {
 1189  0 this.keyLen = keyLen;
 1190  0 dirty = true;
 1191    }
 1192   
 1193    /** The length of the Key */
 1194  0 public synchronized final short getKeyLen() {
 1195  0 return keyLen;
 1196    }
 1197   
 1198    /** The hashed value of the Key for quick comparisons */
 1199  0 public synchronized final void setKeyHash(int keyHash) {
 1200  0 this.keyHash = keyHash;
 1201  0 dirty = true;
 1202    }
 1203   
 1204    /** The hashed value of the Key for quick comparisons */
 1205  55616 public synchronized final int getKeyHash() {
 1206  55616 return keyHash;
 1207    }
 1208   
 1209    /** The length of the Data */
 1210  0 public synchronized final void setDataLen(int dataLen) {
 1211  0 this.dataLen = dataLen;
 1212  0 dirty = true;
 1213    }
 1214   
 1215    /** The length of the Data */
 1216  9190 public synchronized final int getDataLen() {
 1217  9190 return dataLen;
 1218    }
 1219   
 1220    /** The length of the Record's value */
 1221  37329 public synchronized void setRecordLen(int recordLen) {
 1222  37333 this.recordLen = recordLen;
 1223  37333 dirty = true;
 1224    }
 1225   
 1226    /** The length of the Record's value */
 1227  72029 public synchronized final int getRecordLen() {
 1228  72029 return recordLen;
 1229    }
 1230   
 1231    /** The next page for this Record (if overflowed) */
 1232  50678 public synchronized final void setNextPage(long nextPage) {
 1233  50679 this.nextPage = nextPage;
 1234  50680 dirty = true;
 1235    }
 1236   
 1237    /** The next page for this Record (if overflowed) */
 1238  75110 public synchronized final long getNextPage() {
 1239  75110 return nextPage;
 1240    }
 1241    }
 1242   
 1243    /**
 1244    * The object wrapping page number.
 1245    */
 1246    protected static class PageKey implements Comparable {
 1247   
 1248    /**
 1249    * This page number
 1250    */
 1251    protected final long pageNum;
 1252   
 1253   
 1254  197564 public PageKey(long pageNum) {
 1255  197563 this.pageNum = pageNum;
 1256    }
 1257   
 1258    // No synchronization - pageNum is final
 1259  14067 public long getPageNum() {
 1260  14068 return pageNum;
 1261    }
 1262   
 1263    // No synchronization: pageNum is final.
 1264  0 public int compareTo(Object o) {
 1265  0 return (int) (pageNum - ((PageKey) o).pageNum);
 1266    }
 1267   
 1268    /**
 1269    * Return page hash code, which is hash code of its {@link #pageNum}.
 1270    *
 1271    * @return Page hash code
 1272    */
 1273  254647 public int hashCode() {
 1274    // Unroll new Long(pageNum).hashCode()
 1275  254649 return (int) (pageNum ^ (pageNum >> 32));
 1276    }
 1277   
 1278    /**
 1279    * Pages are equal if they are the same or have equal pageNum.
 1280    *
 1281    * @param obj Another page
 1282    * @return true if pages are equal
 1283    */
 1284  89785 public boolean equals(Object obj) {
 1285  89785 if (obj == this) {
 1286  0 return true;
 1287    }
 1288   
 1289  89785 if (obj instanceof PageKey) {
 1290  89717 return pageNum == ((PageKey) obj).pageNum;
 1291    }
 1292   
 1293  65 return false;
 1294    }
 1295    }
 1296   
 1297    /**
 1298    * Paged file's page
 1299    */
 1300    protected final class Page extends PageKey {
 1301   
 1302    /**
 1303    * The Header for this Page
 1304    */
 1305    private final PageHeader header;
 1306   
 1307    /**
 1308    * The offset into the file that this page starts
 1309    */
 1310    private final long offset;
 1311   
 1312    /**
 1313    * The data for this page. Null if page is not loaded.
 1314    */
 1315    private byte[] data;
 1316   
 1317    /**
 1318    * The position (relative) of the Key in the data array
 1319    */
 1320    private int keyPos;
 1321   
 1322    /**
 1323    * The position (relative) of the Data in the data array
 1324    */
 1325    private int dataPos;
 1326   
 1327   
 1328  53196 private Page(long pageNum) {
 1329  53196 super(pageNum);
 1330  53196 this.header = createPageHeader();
 1331  53196 this.offset = fileHeader.headerSize + (pageNum * fileHeader.pageSize);
 1332    }
 1333   
 1334    /**
 1335    * Reads a page into the memory, once. Subsequent calls are ignored.
 1336    */
 1337  118455 public synchronized void read() throws IOException {
 1338  118446 if (data == null) {
 1339  41027 RandomAccessFile raf = null;
 1340  41027 try {
 1341  41027 byte[] data = new byte[fileHeader.pageSize];
 1342  41027 raf = getDescriptor();
 1343  41027 raf.seek(this.offset);
 1344  41027 raf.read(data);
 1345   
 1346    // Read in the header
 1347  41027 ByteArrayInputStream bis = new ByteArrayInputStream(data);
 1348  41027 this.header.read(new DataInputStream(bis));
 1349   
 1350  41027 this.keyPos = fileHeader.pageHeaderSize;
 1351  41027 this.dataPos = this.keyPos + this.header.keyLen;
 1352   
 1353    // Successfully read all the data
 1354  41027 this.data = data;
 1355    } finally {
 1356  41027 putDescriptor(raf);
 1357    }
 1358    }
 1359    }
 1360   
 1361    /**
 1362    * Writes out the header into the this.data, and adds a page to the set of
 1363    * dirty pages.
 1364    */
 1365  41197 public void write() throws IOException {
 1366    // Write out the header into the this.data
 1367  41197 synchronized (this) {
 1368  41198 ByteArrayOutputStream bos = new ByteArrayOutputStream(fileHeader.getPageHeaderSize());
 1369  41196 header.write(new DataOutputStream(bos));
 1370  41199 byte[] b = bos.toByteArray();
 1371  41199 System.arraycopy(b, 0, data, 0, b.length);
 1372    }
 1373   
 1374    // Add to the list of dirty pages
 1375  41198 Paged.this.addDirty(this);
 1376    }
 1377   
 1378    /**
 1379    * Flushes content of the dirty page into the file
 1380    */
 1381  36456 public synchronized void flush() throws IOException {
 1382  36457 RandomAccessFile raf = null;
 1383  36458 try {
 1384  36458 raf = getDescriptor();
 1385  36454 if (this.offset >= raf.length()) {
 1386    // Grow the file
 1387  24 long o = fileHeader.headerSize + (fileHeader.totalCount * 3 / 2 + 1) * fileHeader.pageSize - 1;
 1388  24 raf.seek(o);
 1389  24 raf.writeByte(0);
 1390    }
 1391  36456 raf.seek(this.offset);
 1392  36458 raf.write(this.data);
 1393    } finally {
 1394  36456 putDescriptor(raf);
 1395    }
 1396    }
 1397   
 1398    // No synchronization - header is final
 1399  238387 public PageHeader getPageHeader() {
 1400  238388 return this.header;
 1401    }
 1402   
 1403  3017 public synchronized void setKey(Key key) {
 1404  3016 header.setKey(key);
 1405    // Insert the key into the data array.
 1406  3017 key.copyTo(this.data, this.keyPos);
 1407   
 1408    // Set the start of data to skip over the key.
 1409  3015 this.dataPos = this.keyPos + header.keyLen;
 1410    }
 1411   
 1412  3020 public synchronized Key getKey() {
 1413  3020 if (header.keyLen == 0) {
 1414  0 return null;
 1415    }
 1416   
 1417  3020 return new Key(this.data, this.keyPos, header.keyLen);
 1418    }
 1419   
 1420  40658 public synchronized void streamTo(OutputStream os) throws IOException {
 1421  40658 if (header.dataLen > 0) {
 1422  39507 os.write(this.data, this.dataPos, header.dataLen);
 1423    }
 1424    }
 1425   
 1426  34319 public synchronized void streamFrom(InputStream is) throws IOException {
 1427  34319 int avail = is.available();
 1428  34319 header.dataLen = fileHeader.workSize - header.keyLen;
 1429  34319 if (avail < header.dataLen) {
 1430  34316 header.dataLen = avail;
 1431    }
 1432  34319 if (header.dataLen > 0) {
 1433  32089 is.read(this.data, this.keyPos + header.keyLen, header.dataLen);
 1434    }
 1435    }
 1436    }
 1437    }