Clover coverage report -
Coverage timestamp: Sun Nov 16 2008 23:05:30 GMT
Overview   Package   File
    FRAMES   NO FRAMES  
file stats: LOC: 1,437   Methods: 99
NCLOC: 815   Classes: 5
 
 Source file Conditionals Statements Methods TOTAL
Paged.java 70.9% 81.1% 78.8% 78.9%
coverage coverage
 1    /*
 2    * Licensed to the Apache Software Foundation (ASF) under one or more
 3    * contributor license agreements. See the NOTICE file distributed with
 4    * this work for additional information regarding copyright ownership.
 5    * The ASF licenses this file to You under the Apache License, Version 2.0
 6    * (the "License"); you may not use this file except in compliance with
 7    * the License. You may obtain a copy of the License at
 8    *
 9    * http://www.apache.org/licenses/LICENSE-2.0
 10    *
 11    * Unless required by applicable law or agreed to in writing, software
 12    * distributed under the License is distributed on an "AS IS" BASIS,
 13    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14    * See the License for the specific language governing permissions and
 15    * limitations under the License.
 16    *
 17    * $Id: Paged.java 712683 2008-11-10 15:54:38Z natalia $
 18    */
 19   
 20    package org.apache.xindice.core.filer;
 21   
 22    import org.apache.commons.logging.Log;
 23    import org.apache.commons.logging.LogFactory;
 24    import org.apache.xindice.core.DBException;
 25    import org.apache.xindice.core.DBObject;
 26    import org.apache.xindice.core.FaultCodes;
 27    import org.apache.xindice.core.data.Key;
 28    import org.apache.xindice.core.data.Value;
 29    import org.apache.xindice.util.Configurable;
 30    import org.apache.xindice.util.Configuration;
 31   
 32    import java.io.ByteArrayInputStream;
 33    import java.io.ByteArrayOutputStream;
 34    import java.io.DataInput;
 35    import java.io.DataInputStream;
 36    import java.io.DataOutput;
 37    import java.io.DataOutputStream;
 38    import java.io.File;
 39    import java.io.IOException;
 40    import java.io.InputStream;
 41    import java.io.OutputStream;
 42    import java.io.RandomAccessFile;
 43    import java.lang.ref.WeakReference;
 44    import java.util.Collection;
 45    import java.util.EmptyStackException;
 46    import java.util.HashMap;
 47    import java.util.Iterator;
 48    import java.util.Map;
 49    import java.util.Stack;
 50    import java.util.WeakHashMap;
 51   
 52    /**
 53    * Paged is a paged file implementation that is foundation for both the
 54    * BTree class and the HashFiler. It provides flexible paged I/O and
 55    * page caching functionality.
 56    *
 57    * <br>
 58    * Paged has folowing configuration attributes:
 59    * <ul>
 60    * <li><strong>pagesize</strong>: Size of the page used by the paged file.
 61    * Default page size is 4096 bytes. This parameter can be set only
 62    * before paged file is created. Once it is created, this parameter
 63    * can not be changed.</li>
 64    * <li><strong>pagecount</strong>: Number of pages filer will be created
 65    * with.</li>
 66    * <li><strong>maxkeysize</strong>: Maximum allowed size of the key.
 67    * Default maximum key size is 256 bytes.</li>
 68    * <li><strong>max-descriptors</strong>: Defines maximum amount of
 69    * simultaneously opened file descriptors this paged file can have.
 70    * Several descriptors are needed to provide multithreaded access
 71    * to the underlying file. Too large number will limit amount of
 72    * collections you can open. Default value is 16
 73    * (DEFAULT_DESCRIPTORS_MAX).</li>
 74    * </ul>
 75    *
 76    * <br>FIXME: Currently it seems that maxkeysize is not used anywhere.
 77    * <br>TODO: Introduce Paged interface, implementations.
 78    *
 79    * @version $Revision: 712683 $, $Date: 2008-11-10 15:54:38 +0000 (Mon, 10 Nov 2008) $
 80    */
 81    public abstract class Paged implements DBObject, Configurable {
 82   
 83    private static final Log log = LogFactory.getLog(Paged.class);
 84   
 85    /**
 86    * The maximum number of pages that will be held in the dirty cache.
 87    * Once number reaches the limit, pages are flushed to disk.
 88    */
 89    private static final int MAX_DIRTY_SIZE = 128;
 90   
 91    /**
 92    * Name of the configuration attribute "pagesize"
 93    */
 94    protected static final String CONFIG_PAGESIZE = "pagesize";
 95   
 96    /**
 97    * Name of the configuration attribute "pagecount"
 98    */
 99    protected static final String CONFIG_PAGECOUNT = "pagecount";
 100   
 101    /**
 102    * Name of the configuration attribute "maxkeysize"
 103    */
 104    protected static final String CONFIG_KEYSIZE_MAX = "maxkeysize";
 105   
 106    /**
 107    * Name of the configuration attribute "max-descriptors"
 108    */
 109    protected static final String CONFIG_DESCRIPTORS_MAX = "max-descriptors";
 110   
 111    /**
 112    * Default value of the "pagesize".
 113    */
 114    private static final int DEFAULT_PAGESIZE = 4096;
 115   
 116    /**
 117    * Default value of the "pagecount".
 118    */
 119    private static final int DEFAULT_PAGECOUNT = 1024;
 120   
 121    /**
 122    * File header size
 123    */
 124    private static final int FILE_HEADER_SIZE = 4096;
 125   
 126    /**
 127    * Default value of the "maxkeysize".
 128    */
 129    private static final int DEFAULT_KEYSIZE_MAX = 256;
 130   
 131    /**
 132    * Default value of the maximum number of open random access files paged
 133    * can have. This number balances resources utilization and parallelism of
 134    * access to the paged file.
 135    */
 136    private static final int DEFAULT_DESCRIPTORS_MAX = 16;
 137   
 138   
 139    /**
 140    * Unused page status
 141    */
 142    protected static final byte UNUSED = 0;
 143   
 144    /**
 145    * Overflow page status
 146    */
 147    protected static final byte OVERFLOW = 126;
 148   
 149    /**
 150    * Deleted page status
 151    */
 152    protected static final byte DELETED = 127;
 153   
 154    /**
 155    * Page ID of non-existent page
 156    */
 157    protected static final int NO_PAGE = -1;
 158   
 159   
 160    /**
 161    * Configuration of this paged instance
 162    */
 163    private Configuration config;
 164   
 165    /**
 166    * Map of pages in use. Guarantees that page with same number will be loaded
 167    * into memory just once, allowing to synchronize on page objects to guarantee
 168    * no two threads are writing into same page at once.
 169    *
 170    * <p>Map contains weak references to the Page objects, keys are pages themselves.
 171    * Access is synchronized by the {@link #pagesLock}.
 172    */
 173    private final Map pages = new WeakHashMap();
 174   
 175    /**
 176    * Lock for synchronizing access to the {@link #pages} map.
 177    */
 178    private final Object pagesLock = new Object();
 179   
 180    /**
 181    * Cache of modified pages waiting to be written out.
 182    * Access is synchronized by the {@link #dirtyLock}.
 183    */
 184    private Map dirty = new HashMap();
 185   
 186    /**
 187    * Lock for synchronizing access to the {@link #dirty} map.
 188    */
 189    private final Object dirtyLock = new Object();
 190   
 191    /**
 192    * Random access file descriptors cache.
 193    * Access to it and to {@link #descriptorsCount} is synchronized by itself.
 194    */
 195    private final Stack descriptors = new Stack();
 196   
 197    /**
 198    * The number of random access file objects that exist, either in the
 199    * cache {@link #descriptors}, or currently in use.
 200    */
 201    private int descriptorsCount;
 202   
 203    /**
 204    * The maximum number of random access file objects that can be opened
 205    * by this paged instance.
 206    */
 207    private int descriptorsMax;
 208   
 209    /**
 210    * Whether the file is opened or not.
 211    */
 212    private boolean opened;
 213   
 214    /**
 215    * The underlying file where the Paged object stores its pages.
 216    */
 217    private File file;
 218   
 219    /**
 220    * Header of this Paged
 221    */
 222    private final FileHeader fileHeader;
 223   
 224   
 225  1134 public Paged() {
 226  1134 descriptorsMax = DEFAULT_DESCRIPTORS_MAX;
 227  1134 fileHeader = createFileHeader();
 228    }
 229   
 230  0 public Paged(File file) {
 231  0 this();
 232  0 setFile(file);
 233    }
 234   
 235  1134 public void setConfig(Configuration config) {
 236  1134 this.config = config;
 237    // Read paged config
 238  1134 fileHeader.setPageSize(config.getIntAttribute(CONFIG_PAGESIZE,
 239    fileHeader.getPageSize()));
 240  1134 fileHeader.setPageCount(config.getLongAttribute(CONFIG_PAGECOUNT,
 241    fileHeader.getPageCount()));
 242  1134 fileHeader.setMaxKeySize(config.getShortAttribute(CONFIG_KEYSIZE_MAX,
 243    fileHeader.getMaxKeySize()));
 244  1134 descriptorsMax = getConfig().getIntAttribute(CONFIG_DESCRIPTORS_MAX,
 245    descriptorsMax);
 246    }
 247   
 248  1134 public Configuration getConfig() {
 249  1134 return config;
 250    }
 251   
 252    /**
 253    * setFile sets the file object for this Paged.
 254    *
 255    * @param file The File
 256    */
 257  1134 protected final void setFile(final File file) {
 258  1134 this.file = file;
 259    }
 260   
 261    /**
 262    * getFile returns the file object for this Paged.
 263    *
 264    * @return The File
 265    */
 266  849 protected final File getFile() {
 267  849 return file;
 268    }
 269   
 270    /**
 271    * Obtain RandomAccessFile ('descriptor') object out of the pool.
 272    * If no descriptors available, and maximum amount already allocated,
 273    * the call will block.
 274    */
 275  72826 protected final RandomAccessFile getDescriptor() throws IOException {
 276  72826 synchronized (descriptors) {
 277    // If there are descriptors in the cache return one.
 278  72826 if (!descriptors.empty()) {
 279  68803 return (RandomAccessFile) descriptors.pop();
 280    }
 281    // Otherwise we need to get one some other way.
 282   
 283    // First try to create a new one if there's room
 284  4023 if (descriptorsCount < descriptorsMax) {
 285  4023 descriptorsCount++;
 286  4023 return new RandomAccessFile(file, "rw");
 287    }
 288   
 289    // Otherwise we have to wait for one to be released by another thread.
 290  0 while (true) {
 291  0 try {
 292  0 descriptors.wait();
 293  0 return (RandomAccessFile) descriptors.pop();
 294    } catch (InterruptedException e) {
 295    // Ignore, and continue to wait
 296    } catch (EmptyStackException e) {
 297    // Ignore, and continue to wait
 298    }
 299    }
 300    }
 301    }
 302   
 303    /**
 304    * Puts a RandomAccessFile ('descriptor') back into the descriptor pool.
 305    */
 306  72826 protected final void putDescriptor(RandomAccessFile raf) {
 307  72825 if (raf != null) {
 308  72825 synchronized (descriptors) {
 309  72826 descriptors.push(raf);
 310  72826 descriptors.notify();
 311    }
 312    }
 313    }
 314   
 315    /**
 316    * Closes a RandomAccessFile ('descriptor') and removes it from the pool.
 317    */
 318  4019 protected final void closeDescriptor(RandomAccessFile raf) {
 319  4023 if (raf != null) {
 320  4023 try {
 321  4023 raf.close();
 322    } catch (IOException e) {
 323    // Ignore close exception
 324    }
 325   
 326    // Synchronization is necessary as decrement operation is not atomic
 327  4023 synchronized (descriptors) {
 328  4023 descriptorsCount --;
 329    }
 330    }
 331    }
 332   
 333    /**
 334    * getPage returns the page specified by pageNum.
 335    *
 336    * @param pageNum The Page number
 337    * @return The requested Page
 338    * @throws IOException if an Exception occurs
 339    */
 340  87637 protected final Page getPage(long pageNum) throws IOException {
 341  87637 Page p = updateIdentityMap(pageNum);
 342   
 343    // Load the page from disk if necessary
 344  87637 p.read();
 345  87637 return p;
 346    }
 347   
 348  98300 private Page updateIdentityMap(long pageNum) {
 349  98299 final PageKey k = new PageKey(pageNum);
 350  98301 Page p = null;
 351  98304 synchronized (pagesLock) {
 352    // Check if page is already loaded in the page cache
 353  98307 WeakReference ref = (WeakReference) pages.get(k);
 354  98307 if (ref != null) {
 355  68956 p = (Page) ref.get();
 356    // Fall through to p.read(). Even if page present in the pages
 357    // map, it still has to be read - it could be that it was just
 358    // added to the map but read() was not called yet.
 359    }
 360   
 361    // If not found, create it and add it to the pages cache
 362  98307 if (p == null) {
 363  29351 p = new Page(pageNum);
 364  29351 pages.put(p, new WeakReference(p));
 365    }
 366    }
 367   
 368  98303 return p;
 369    }
 370   
 371    /**
 372    * readValue reads the multi-Paged Value starting at the specified
 373    * Page.
 374    *
 375    * @param page The starting Page
 376    * @return The Value
 377    * @throws IOException if an Exception occurs
 378    */
 379  13301 protected final Value readValue(Page page) throws IOException {
 380  13301 final PageHeader sph = page.getPageHeader();
 381  13301 ByteArrayOutputStream bos = new ByteArrayOutputStream(sph.getRecordLen());
 382   
 383    // Loop until we've read all the pages into memory.
 384  13301 Page p = page;
 385  13301 while (true) {
 386  13302 PageHeader ph = p.getPageHeader();
 387   
 388    // Add the contents of the page onto the stream
 389  13302 p.streamTo(bos);
 390   
 391    // Continue following the list of pages until we get to the end.
 392  13302 long nextPage = ph.getNextPage();
 393  13302 if (nextPage == NO_PAGE) {
 394  13301 break;
 395    }
 396  1 p = getPage(nextPage);
 397    }
 398   
 399    // Return a Value with the collected contents of all pages.
 400  13301 return new Value(bos.toByteArray());
 401    }
 402   
 403    /**
 404    * readValue reads the multi-Paged Value starting at the specified
 405    * page number.
 406    *
 407    * @param page The starting page number
 408    * @return The Value
 409    * @throws IOException if an Exception occurs
 410    */
 411  0 protected final Value readValue(long page) throws IOException {
 412  0 return readValue(getPage(page));
 413    }
 414   
 415    /**
 416    * writeValue writes the multi-Paged Value starting at the specified
 417    * Page.
 418    *
 419    * @param page The starting Page
 420    * @param value The Value to write
 421    * @throws IOException if an Exception occurs
 422    */
 423  28162 protected final void writeValue(Page page, Value value) throws IOException {
 424  28161 if (value == null) {
 425  0 throw new IOException("Can't write a null value");
 426    }
 427   
 428  28162 InputStream is = value.getInputStream();
 429   
 430    // Write as much as we can onto the primary page.
 431  28160 PageHeader hdr = page.getPageHeader();
 432  28161 hdr.setRecordLen(value.getLength());
 433  28162 page.streamFrom(is);
 434   
 435    // Write out the rest of the value onto any needed overflow pages
 436  28162 while (is.available() > 0) {
 437  3 Page lpage = page;
 438  3 PageHeader lhdr = hdr;
 439   
 440    // Find an overflow page to use
 441  3 long np = lhdr.getNextPage();
 442  3 if (np != NO_PAGE) {
 443    // Use an existing page.
 444  0 page = getPage(np);
 445    } else {
 446    // Create a new overflow page
 447  3 page = getFreePage();
 448  3 lhdr.setNextPage(page.getPageNum());
 449    }
 450   
 451    // Mark the page as an overflow page.
 452  3 hdr = page.getPageHeader();
 453  3 hdr.setStatus(OVERFLOW);
 454   
 455    // Write some more of the value to the overflow page.
 456  3 page.streamFrom(is);
 457  3 lpage.write();
 458    }
 459   
 460    // Cleanup any unused overflow pages. i.e. the value is smaller then the
 461    // last time it was written.
 462  28162 long np = hdr.getNextPage();
 463  28160 if (np != NO_PAGE) {
 464  0 unlinkPages(np);
 465    }
 466   
 467  28160 hdr.setNextPage(NO_PAGE);
 468  28162 page.write();
 469    }
 470   
 471    /**
 472    * writeValue writes the multi-Paged Value starting at the specified
 473    * page number.
 474    *
 475    * @param page The starting page number
 476    * @param value The Value to write
 477    * @throws IOException if an Exception occurs
 478    */
 479  0 protected final void writeValue(long page, Value value) throws IOException {
 480  0 writeValue(getPage(page), value);
 481    }
 482   
 483    /**
 484    * unlinkPages unlinks a set of pages starting at the specified Page.
 485    *
 486    * @param page The starting Page to unlink
 487    * @throws IOException if an Exception occurs
 488    */
 489  3117 protected void unlinkPages(Page page) throws IOException {
 490    // Add any overflow pages to the list of free pages.
 491    // Get the first and last page in the chain.
 492  3117 long firstPage = page.pageNum;
 493  3117 while (page.header.nextPage != NO_PAGE) {
 494  0 page = getPage(page.header.nextPage);
 495    }
 496  3117 long lastPage = page.pageNum;
 497   
 498    // Free the chain
 499  3117 synchronized (fileHeader) {
 500    // If there are already some free pages, add the start of the chain
 501    // to the list of free pages.
 502