Examples of BigArrayImpl

com.leansoft.bigqueue.BigArrayImpl
A big array implementation supporting sequential append and random read. Main features: 1. FAST : close to the speed of direct memory access, extremely fast in append only and sequential read modes, sequential append and read are close to O(1) memory access, random read is close to O(1) memory access if data is in cache and is close to O(1) disk access if data is not in cache. 2. MEMORY-EFFICIENT : automatic paging & swapping algorithm, only most-recently accessed data is kept in memory. 3. THREAD-SAFE : multiple threads can concurrently read/append the array without data corruption. 4. PERSISTENT - all array data is persisted on disk, and is crash resistant. 5. BIG(HUGE) - the total size of the array data is only limited by the available disk space. @author bulldog

Examples of com.leansoft.bigqueue.BigArrayImpl

  }
  
  @Test
  public void testInvalidDataPageSize() throws IOException {
    try {
      bigArray = new BigArrayImpl(testDir, "invalid_data_page_size", BigArrayImpl.MINIMUM_DATA_PAGE_SIZE - 1);
      fail("should throw invalid page size exception");
    } catch (IllegalArgumentException iae) {
      // ecpected
    }
  }

View Full Code Here

Examples of com.leansoft.bigqueue.BigArrayImpl

    }
  }
  
  @Test
  public void testMinimumDataPageSize() throws IOException {
    bigArray = new BigArrayImpl(testDir, "min_data_page_size", BigArrayImpl.MINIMUM_DATA_PAGE_SIZE);
    
    String randomString = TestUtil.randomString(BigArrayImpl.MINIMUM_DATA_PAGE_SIZE / (1024 * 1024));
    
    for(int i = 0; i < 1024 * 1024; i++) {
      bigArray.append(randomString.getBytes());

View Full Code Here

Examples of com.leansoft.bigqueue.BigArrayImpl

    // first sorted item
    String previousItem = new String(targetSortedQueue.dequeue());
    
    // put sorted items in a big array so we can binary search it later
    // validate the sorted queue at the same time
    IBigArray bigArray = new BigArrayImpl(MergeSortHelper.SAMPLE_DIR, "sample_array");
    bigArray.append(previousItem.getBytes());
    for(int i = 1; i < targetSize; i++) {
      String item = new String(targetSortedQueue.dequeue());
      if (item.compareTo(previousItem) < 0) {
        System.err.println("target queue is not in sorted order!");
      }
      bigArray.append(item.getBytes());
      previousItem = item;
    }
    MergeSortHelper.output("Validation finished.");
    
    // have done with target sorted queue, empty it and delete back data files to save disk space
    targetSortedQueue.removeAll();
    targetSortedQueue.close();
    
    bigArray.close();
  }

View Full Code Here

Examples of com.leansoft.bigqueue.BigArrayImpl

    
    bigArray.close();
  }
  
  static void binarySearch() throws IOException {
    IBigArray bigArray = new BigArrayImpl(MergeSortHelper.SAMPLE_DIR, "sample_array"); 
    // randomly get some items which can be found later
    int searchLoopLimit = 10000;
    List<String> randomKeys = new ArrayList<String>();
    Random random = new Random();
    for(int i = 0; i < searchLoopLimit / 2; i++) {
      long randomIndex = (long)(random.nextDouble() * maxNumOfItems);
      String randomKey = new String(bigArray.get(randomIndex));
      randomKeys.add(randomKey);
    }
    bigArray.close(); // close to release cache, otherwise the performance of binary search will not be real.
    
    bigArray = new BigArrayImpl(MergeSortHelper.SAMPLE_DIR, "sample_array"); 
    
    MergeSortHelper.output("Single thread binary search begin ...");
    int foundCount = 0;
    long start = System.currentTimeMillis();
    for(int i = 0; i < searchLoopLimit; i++) {
      String randomKey = null;
      if (i < searchLoopLimit / 2) {
        randomKey = randomKeys.get(i);
      } else {
        randomKey = MergeSortHelper.genRandomString(itemSize);
      }
      long result = BinarySearchHelper.binarySearch(bigArray, randomKey, 0, bigArray.size() - 1);
      if (result != BinarySearchHelper.NOT_FOUND) {
        foundCount++;
      }
    }
    long end = System.currentTimeMillis();
    MergeSortHelper.output("Binary search finished.");
    MergeSortHelper.output("Time to search " + searchLoopLimit + " items in the big array is " + (end - start) + " ms.");
    MergeSortHelper.output("Average search time is " + (double)(end - start) / searchLoopLimit + "ms.");
    MergeSortHelper.output("Found count is " + foundCount);
    
    // have done with the big array, empty it and delete back data files to save disk space
    bigArray.removeAll();
    bigArray.close();
  }

View Full Code Here

Examples of com.leansoft.bigqueue.BigArrayImpl

    }
  }
  
  @Test
  public void runTest() throws Exception {
    bigArray = new BigArrayImpl(testDir, "load_test");
    
    System.out.println("Load test begin ...");
    for(int i = 0; i < loop; i++) {
      System.out.println("[doRunProduceThenConsume] round " + (i + 1) + " of " + loop);
      this.doRunProduceThenConsume();
      
      // reset
      producingItemCount.set(0);
      itemMap.clear();
      bigArray.removeAll();
    }
    
    bigArray.close();
    bigArray = new BigArrayImpl(testDir, "load_test");
    
    for(int i = 0; i < loop; i++) {
      System.out.println("[doRunMixed] round " + (i + 1) + " of " + loop);
      this.doRunMixed();

View Full Code Here

Examples of com.leansoft.bigqueue.BigArrayImpl

  @Test
  public void demo() throws IOException {
    IBigArray bigArray = null;
    try {
      // create a new big array
      bigArray = new BigArrayImpl("d:/bigarray/tutorial", "demo");
      // ensure the new big array is empty
      assertNotNull(bigArray);
      assertTrue(bigArray.isEmpty());
      assertTrue(bigArray.size() == 0);
      assertTrue(bigArray.getHeadIndex() == 0);

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.