Examples of org.apache.hama.HamaConfiguration

org.apache.hama.HamaConfiguration
Adds Hama configuration files to a Configuration.


  @Override
  protected void setUp() throws Exception {


    super.setUp();
    conf = new HamaConfiguration();


    conf.setInt(Constants.GROOM_PING_PERIOD, 200);
    conf.setClass("bsp.work.class", FaulTestBSP.class, BSP.class);
    conf.setClass(SyncServiceFactory.SYNC_PEER_CLASS,
        LocalBSPRunner.LocalSyncClient.class, SyncClient.class);

View Full Code Here


    public static void main(String[] args) throws Throwable {
      if (LOG.isDebugEnabled())
        LOG.debug("BSPPeerChild starting");


      final HamaConfiguration defaultConf = new HamaConfiguration();
      // report address
      String host = args[0];
      int port = Integer.parseInt(args[1]);
      InetSocketAddress address = new InetSocketAddress(host, port);
      TaskAttemptID taskid = TaskAttemptID.forName(args[2]);


      // //////////////////
      BSPPeerProtocol umbilical = (BSPPeerProtocol) RPC.getProxy(
          BSPPeerProtocol.class, HamaRPCProtocolVersion.versionID, address,
          defaultConf);


      final BSPTask task = (BSPTask) umbilical.getTask(taskid);
      int peerPort = umbilical.getAssignedPortNum(taskid);


      defaultConf.addResource(new Path(task.getJobFile()));
      BSPJob job = new BSPJob(task.getJobID(), task.getJobFile());


      defaultConf.set(Constants.PEER_HOST, args[3]);
      if (null != args && 5 == args.length) {
        defaultConf.setInt("bsp.checkpoint.port", Integer.parseInt(args[4]));
      }
      defaultConf.setInt(Constants.PEER_PORT, peerPort);


      long superstep = Long.parseLong(args[4]);
      TaskStatus.State state = TaskStatus.State.valueOf(args[5]);
      LOG.debug("Starting peer for step " + superstep + " state = " + state);

View Full Code Here


        Path localJarFile = defaultJobConf.getLocalPath(SUBDIR + "/"
            + task.getTaskID() + "/" + "job.jar");
        dfs.copyToLocalFile(new Path(task.getJobFile()), localJobFile);


        HamaConfiguration conf = new HamaConfiguration();
        conf.addResource(localJobFile);
        jobConf = new BSPJob(conf, task.getJobID().toString());


        Path jarFile = null;
        if (jobConf.getJar() != null) {
          jarFile = new Path(jobConf.getJar());
        } else {
          LOG.warn("No jar file for job " + task.getJobID()
              + " has been defined!");
        }
        jobConf.setJar(localJarFile.toString());


        if (jarFile != null) {
          dfs.copyToLocalFile(jarFile, localJarFile);


          // also unjar the job.jar files in workdir
          File workDir = new File(
              new File(localJobFile.toString()).getParent(), "work");
          if (!workDir.mkdirs()) {
            if (!workDir.isDirectory()) {
              throw new IOException("Mkdirs failed to create "
                  + workDir.toString());
            }
          }
          RunJar.unJar(new File(localJarFile.toString()), workDir);
        }
        rjob.localized = true;
      } else {
        HamaConfiguration conf = new HamaConfiguration();
        conf.addResource(rjob.getJobFile());
        jobConf = new BSPJob(conf, rjob.getJobId().toString());
      }
    }


    launchTaskForJob(tip, jobConf);

View Full Code Here

    
    cluster = controller.launchCluster(clusterSpec);
    proxy = new HadoopProxy(clusterSpec, cluster);
    proxy.start();


    HamaConfiguration conf = getConfiguration();
    BSPJobClient client = new BSPJobClient(conf);
    waitForGroomServers(client);
    
    running = true;
  }

View Full Code Here

      }
    }
  }


  public HamaConfiguration getConfiguration() {
    HamaConfiguration conf = new HamaConfiguration();
    for (Entry<Object, Object> entry : cluster.getConfiguration().entrySet()) {
      conf.set(entry.getKey().toString(), entry.getValue().toString());
    }
    return conf;
  }

View Full Code Here

    controller.shutdown();
  }


  @Test(timeout = TestConstants.ITEST_TIMEOUT)
  public void test() throws Exception {
    HamaConfiguration jobConf = controller.getConfiguration();
    jobConf.set("hadoop.rpc.socket.factory.class.default",
        "org.apache.hadoop.net.StandardSocketFactory");


    BSPJob bsp = new BSPJob(jobConf, new BSPJobID());
    LOG.info("Job conf: "
        + bsp.getConf().get("hadoop.rpc.socket.factory.class.default") + ", "
        + bsp.getJobID().toString());


    bsp.setJarByClass(MyEstimator.class);
    bsp.setBspClass(MyEstimator.class);
    bsp.setInputFormat(NullInputFormat.class);
    bsp.setOutputKeyClass(Text.class);
    bsp.setOutputValueClass(DoubleWritable.class);
    bsp.setOutputFormat(TextOutputFormat.class);
    bsp.set("bsp.working.dir", "/tmp");
    FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);


    LOG.info("Client configuration start ..");


    HamaConfiguration clientConf = controller.getConfiguration();
    BSPJobClient jobClient = new BSPJobClient(clientConf);
    ClusterStatus cluster = jobClient.getClusterStatus(true);
    assertNotNull(cluster);
    assertTrue(cluster.getGroomServers() > 0);
    assertTrue(cluster.getMaxTasks() > 1);
    bsp.setNumBspTask(cluster.getMaxTasks());


    LOG.info("Client conf: "
        + clientConf.get("hadoop.rpc.socket.factory.class.default"));


    RunningJob rJob = jobClient.submitJob(bsp);
    rJob.waitForCompletion();
    LOG.info("finished");
  }

View Full Code Here

   * Run the tool.
   * 
   * @param args Command line arguments. First arg is path to zookeepers file.
   */
  public static void main(String args[]) {
    Configuration conf = new HamaConfiguration();
    // Note that we do not simply grab the property ZOOKEEPER_QUORUM from
    // the HamaConfiguration because the user may be using a zoo.cfg file.
    Properties zkProps = QuorumPeer.makeZKProps(conf);
    for (Entry<Object, Object> entry : zkProps.entrySet()) {
      String key = entry.getKey().toString().trim();

View Full Code Here

  /**
   * Parse ZooKeeper configuration from Hama XML config and run a QuorumPeer.
   * @param baseConf Hadoop Configuration.
   */
  public static void run(Configuration baseConf) {
    Configuration conf = new HamaConfiguration(baseConf);
    try {
      Properties zkProperties = makeZKProps(conf);
      writeMyID(zkProperties);
      QuorumPeerConfig zkConfig = new QuorumPeerConfig();
      zkConfig.parseProperties(zkProperties);

View Full Code Here

  }


  private static void writeMyID(Properties properties) throws IOException {
    long myId = -1;


    Configuration conf = new HamaConfiguration();
    String myAddress = DNS.getDefaultHost(
        conf.get("hama.zookeeper.dns.interface","default"),
        conf.get("hama.zookeeper.dns.nameserver","default"));


    List<String> ips = new ArrayList<String>();


    // Add what could be the best (configured) match
    ips.add(myAddress.contains(".") ?

View Full Code Here

    this.conf = conf;
    this.jobId = jobId;
  }
  
  public BSPJobContext(Path config, BSPJobID jobId) throws IOException {
    this.conf = new HamaConfiguration();
    this.jobId = jobId;
    this.conf.addResource(config);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hama.HamaConfiguration

org.apache.hama.bsp.BSPApplicationMaster

org.apache.hama.bsp.BSPJob

org.apache.hama.bsp.BSPJobClient

org.apache.hama.bsp.BSPJobContext

org.apache.hama.bsp.BSPRunner

org.apache.hama.bsp.GroomServer

org.apache.hama.bsp.GroomServer$BSPPeerChild

org.apache.hama.bsp.GroomServer$Child

org.apache.hama.bsp.LocalBSPRunner

org.apache.hama.bsp.MesosExecutor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.