Examples of BigQueueImpl


Examples of com.leansoft.bigqueue.BigQueueImpl

    if (bigQueue == null) {
      synchronized(lock) {
        bigQueue = queueMap.get(topic);
        if (bigQueue == null) {
          try {
            bigQueue = new BigQueueImpl(queueDir, topic);
            queueMap.put(topic, bigQueue);
          } catch (IOException e) {
            resp.setResultCode(ResultCode.FAILURE);
            return resp;
          }
View Full Code Here

Examples of com.leansoft.bigqueue.BigQueueImpl

    MergeSortHelper.SAMPLE_DIR = MergeSortHelper.SAMPLE_DIR + "multi_thread";
   
    MergeSortHelper.output("Multi threads sort begin ...");
     
    MergeSortHelper.output("Generating random big queue ...");
        IBigQueue srcBigQueue = new BigQueueImpl(MergeSortHelper.SAMPLE_DIR, "srcq");
       
        Populator[] populators = new Populator[threadNum];
        for(int i = 0; i < threadNum; i++) {
          populators[i] = new Populator(srcBigQueue, maxNumOfItems, itemSize);
          populators[i].start();
        }
        for(int i = 0; i < threadNum; i++) {
          try {
        populators[i].join();
      } catch (InterruptedException e) {
        // ignore
      }
        }
       
        long start = System.currentTimeMillis();
        MergeSortHelper.output("Making queue of sorted queues ...");
      Queue<IBigQueue> queueOfSortedQueues = new LinkedBlockingQueue<IBigQueue>();
        SortedQueueMaker[] sortedQueueMakers = new SortedQueueMaker[threadNum];
        for(int i = 0; i < threadNum; i++) {
          sortedQueueMakers[i] = new SortedQueueMaker(srcBigQueue, maxInMemSortNumOfItems, queueOfSortedQueues);
          sortedQueueMakers[i].start();
        }
        for(int i = 0; i < threadNum; i++) {
          try {
            sortedQueueMakers[i].join();
      } catch (InterruptedException e) {
        // ignore
      }
        }
        srcBigQueue.removeAll();
        srcBigQueue.close();
       
    MergeSortHelper.output("Merging and sorting the queues ...");
        MergeSorter[] mergeSorters = new MergeSorter[threadNum];
        for(int i = 0; i < threadNum; i++) {
          mergeSorters[i] = new MergeSorter(queueOfSortedQueues, maxMergeSortWays);
View Full Code Here

Examples of com.leansoft.bigqueue.BigQueueImpl

        String[] stringArray = list.toArray(new String[0]);
        if (stringArray.length == 0) break;
        // sort in memory
        inMemSort(stringArray);
        String newQueueName = getNextTempQueueName();
        IBigQueue newBigQueue = new BigQueueImpl(SAMPLE_DIR, newQueueName);
        // make sorted sub-queue
        for(String item : stringArray) {
          newBigQueue.enqueue(item.getBytes());
        }
        // put the sub-queue into output queue
        queueOfSortedQueues.offer(newBigQueue);
        newBigQueue.close();
       
        if (data == null) break;
        list.clear();
        long end = System.currentTimeMillis();
        output("Time used to make one sorted queue " + (end - begin) + " ms, maxInMemSortNumOfItems = " + maxInMemSortNumOfItems);
View Full Code Here

Examples of com.leansoft.bigqueue.BigQueueImpl

   * @return target sorted queue
   * @throws IOException exception thrown if there is IO error during the operation
   */
  public static IBigQueue nWayMergeSort(List<IBigQueue> listOfSortedQueues) throws IOException {
    String newQueueName = getNextTempQueueName();
    IBigQueue targetBigQueue = new BigQueueImpl(SAMPLE_DIR, newQueueName); // target queue
   
    int ways = listOfSortedQueues.size();
    long begin = System.currentTimeMillis();
    while(true) {
      IBigQueue queueWithLowestItem = null;
      String lowestItem = null;
      // find the lowest item in all n way queues
      for(IBigQueue bigQueue : listOfSortedQueues) {
        if (!bigQueue.isEmpty()) {
          if (queueWithLowestItem == null) {
            queueWithLowestItem = bigQueue;
            lowestItem = new String(bigQueue.peek());
          } else {
            String item = new String(bigQueue.peek());
            if (item.compareTo(lowestItem) < 0) {
              queueWithLowestItem = bigQueue;
              lowestItem = item;
            }
          }
        }
      }
      if (queueWithLowestItem == null) break; // all queues are empty
      // extract and put the lowest item into the target queue
      byte[] data = queueWithLowestItem.dequeue();
      targetBigQueue.enqueue(data);
    }
   
    // release the source queues since we have done with them
    for(IBigQueue bigQueue : listOfSortedQueues) {
      bigQueue.removeAll(); // make empty the queue, delete back data files to save disk space
      bigQueue.close();
    }
   
    targetBigQueue.close();
   
    long end = System.currentTimeMillis();
   
    output("Time used to merge sort  " + ways + " way queues : " + (end - begin) + " ms.");
   
View Full Code Here

Examples of com.leansoft.bigqueue.BigQueueImpl

  }
 
  static void mergeSort() throws IOException {
    MergeSortHelper.output("Single thread sort begin ...");
    MergeSortHelper.output("Generating random big queue ...");
    IBigQueue srcBigQueue = new BigQueueImpl(MergeSortHelper.SAMPLE_DIR, "srcq");
    MergeSortHelper.populateBigQueue(srcBigQueue, maxNumOfItems, itemSize);
   
    long start = System.currentTimeMillis();
    MergeSortHelper.output("Making queue of sorted queues ...");
    Queue<IBigQueue> queueOfSortedQueues = new LinkedList<IBigQueue>();
    MergeSortHelper.makeQueueOfSortedQueues(srcBigQueue, maxInMemSortNumOfItems, queueOfSortedQueues);
    // have done with source queue, empty it and delete back data files to save disk space
    srcBigQueue.removeAll();
    srcBigQueue.close();
   
    MergeSortHelper.output("Merging and sorting the queues ...");
    MergeSortHelper.mergeSort(queueOfSortedQueues, maxMergeSortWays);
    long end = System.currentTimeMillis();
    MergeSortHelper.output("Mergesort finished.");
View Full Code Here

Examples of com.leansoft.bigqueue.BigQueueImpl

   
  }
 
  @Test
  public void runTest() throws Exception {
    bigQueue = new BigQueueImpl(testDir, "load_test");
   
    System.out.println("Load test begin ...");
    for(int i = 0; i < loop; i++) {
      System.out.println("[doRunProduceThenConsume] round " + (i + 1) + " of " + loop);
      this.doRunProduceThenConsume();
     
      // reset
      producingItemCount.set(0);
      consumingItemCount.set(0);
      bigQueue.gc();
    }
   
    bigQueue.close();
    bigQueue = new BigQueueImpl(testDir, "load_test");
   
    for(int i = 0; i < loop; i++) {
      System.out.println("[doRunMixed] round " + (i + 1) + " of " + loop);
      this.doRunMixed();
     
View Full Code Here

Examples of com.leansoft.bigqueue.BigQueueImpl

   
    IBigQueue bigQueue = null;
   
    try {
      // create a new big queue
      bigQueue = new BigQueueImpl("d:/bigqueue/tutorial", "demo");
       
      // ensure the new big queue is empty
      assertNotNull(bigQueue);
      assertTrue(bigQueue.size() == 0);
      assertTrue(bigQueue.isEmpty());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.