Examples of org.apache.accumulo.server.zookeeper.DistributedWorkQueue$Processor

Package org.apache.accumulo.server.zookeeper

Examples of org.apache.accumulo.server.zookeeper.DistributedWorkQueue$Processor

org.apache.accumulo.server.zookeeper.DistributedWorkQueue
Provides a way to push work out to tablet servers via zookeeper and wait for that work to be done. Any tablet server can pick up a work item and process it. Worker processes watch a zookeeper node for tasks to be performed. After getting an exclusive lock on the node, the worker will perform the task.

  public RecoveryManager(Master master) {
    this.master = master;
    executor = Executors.newScheduledThreadPool(4, new NamingThreadFactory("Walog sort starter "));
    zooCache = new ZooCache();
    try {
      List<String> workIDs = new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).getWorkQueued();
      sortsQueued.addAll(workIDs);
    } catch (Exception e) {
      log.warn(e, e);
    }
  }

View Full Code Here


  }


  private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException, IOException {
    String work = source + "|" + destination;
    new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZRECOVERY).addWork(sortId, work.getBytes(Constants.UTF8));


    synchronized (this) {
      sortsQueued.add(sortId);
    }

View Full Code Here

    this.threadPool = new SimpleThreadPool(threadPoolSize, this.getClass().getName());
  }


  public void startWatchingForRecoveryLogs(ThreadPoolExecutor distWorkQThreadPool) throws KeeperException, InterruptedException {
    this.threadPool = distWorkQThreadPool;
    new DistributedWorkQueue(ZooUtil.getRoot(instance) + Constants.ZRECOVERY).startProcessing(new LogProcessor(), this.threadPool);
  }

View Full Code Here

    }
    announceExistence();


    ThreadPoolExecutor distWorkQThreadPool = new SimpleThreadPool(getSystemConfiguration().getCount(Property.TSERV_WORKQ_THREADS), "distributed work queue");


    bulkFailedCopyQ = new DistributedWorkQueue(ZooUtil.getRoot(instance) + Constants.ZBULK_FAILED_COPYQ);
    try {
      bulkFailedCopyQ.startProcessing(new BulkFailedCopyProcessor(), distWorkQThreadPool);
    } catch (Exception e1) {
      throw new RuntimeException("Failed to start distributed work queue for copying ", e1);
    }

View Full Code Here

      fs.rename(orig, dest);
      log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
    }
    
    if (loadedFailures.size() > 0) {
      DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
          + Constants.ZBULK_FAILED_COPYQ);
      
      HashSet<String> workIds = new HashSet<String>();
      
      for (String failure : loadedFailures.values()) {
        Path orig = new Path(failure);
        Path dest = new Path(error, orig.getName());
        
        if (fs.exists(dest))
          continue;
        
        bifCopyQueue.addWork(orig.getName(), (failure + "," + dest).getBytes(Constants.UTF8));
        workIds.add(orig.getName());
        log.debug("tid " + tid + " added to copyq: " + orig + " to " + dest + ": failed");
      }
      
      bifCopyQueue.waitUntilDone(workIds);
    }
    
    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
    return new CleanUpBulkImport(tableId, source, bulk, error);
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.accumulo.server.zookeeper.DistributedWorkQueue$Processor

aurelienribon.ui.css.Property

com.google.walkaround.util.server.RetryHelper

com.webobjects.eocontrol.EOEnterpriseObject

dk.brics.automaton.Automaton

org.apache.accumulo.master.recovery.RecoveryManager

org.apache.accumulo.master.tableOps.CopyFailed

org.apache.accumulo.server.master.recovery.RecoveryManager

org.apache.accumulo.server.master.tableOps.CopyFailed

org.apache.accumulo.server.tabletserver.log.LogSorter

org.apache.accumulo.server.tabletserver.TabletServer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.