Package org.apache.tajo.conf

Examples of org.apache.tajo.conf.TajoConf


    HelpFormatter formatter = new HelpFormatter();
    formatter.printHelp( "tajo_dump [options] [table_name]", options );
  }

  public static void main(String [] args) throws ParseException, IOException, ServiceException, SQLException {
    TajoConf conf = new TajoConf();

    CommandLineParser parser = new PosixParser();
    CommandLine cmd = parser.parse(options, args);

    String hostName = null;
    Integer port = null;
    if (cmd.hasOption("h")) {
      hostName = cmd.getOptionValue("h");
    }
    if (cmd.hasOption("p")) {
      port = Integer.parseInt(cmd.getOptionValue("p"));
    }

    // if there is no "-h" option,
    if(hostName == null) {
      if (conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) {
        // it checks if the client service address is given in configuration and distributed mode.
        // if so, it sets entryAddr.
        hostName = conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0];
      }
    }
    if (port == null) {
      if (conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) {
        // it checks if the client service address is given in configuration and distributed mode.
        // if so, it sets entryAddr.
        port = Integer.parseInt(conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]);
      }
    }

    TajoClient client = null;
    if ((hostName == null) ^ (port == null)) {
      System.err.println("ERROR: cannot find valid Tajo server address");
      System.exit(-1);
    } else if (hostName != null && port != null) {
      conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port);
      client = new TajoClient(conf);
    } else if (hostName == null && port == null) {
      client = new TajoClient(conf);
    }
View Full Code Here


    return queryMaster;
  }

  @Override
  public void init(Configuration conf) {
    TajoConf tajoConf = (TajoConf) conf;
    try {
      // Setup RPC server
      InetSocketAddress initIsa =
          new InetSocketAddress("0.0.0.0", port);
      if (initIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initIsa);
      }

      this.rpcServer = new AsyncRpcServer(TajoWorkerProtocol.class, this, initIsa);
      this.rpcServer.start();

      this.bindAddr = NetUtils.getConnectAddress(rpcServer.getListenAddress());
      this.addr = bindAddr.getHostName() + ":" + bindAddr.getPort();

      this.port = bindAddr.getPort();

      queryMaster = new QueryMaster(workerContext);
      addService(queryMaster);

    } catch (Exception e) {
      LOG.error(e.getMessage(), e);
    }
    // Get the master address
    LOG.info("TajoWorkerManagerService is bind to " + addr);
    tajoConf.setVar(TajoConf.ConfVars.WORKER_PEER_RPC_ADDRESS, NetUtils.normalizeInetSocketAddress(bindAddr));
    super.init(tajoConf);
  }
View Full Code Here

  @Override
  public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext, int numTasks) {
    int numClusterNodes = workerContext.getNumClusterNodes();

    TajoConf conf =  (TajoConf)workerContext.getQueryMaster().getConfig();
    int workerNum = conf.getIntVar(TajoConf.ConfVars.YARN_RM_WORKER_NUMBER_PER_NODE);
    return numClusterNodes == 0 ? numTasks: Math.min(numTasks, numClusterNodes * workerNum);
  }
View Full Code Here

  public void startTask(final String[] params) {
    //TODO change to use event dispatcher
    Thread t = new Thread() {
      public void run() {
        try {
          TajoConf systemConf = new TajoConf(tajoConf);
          TaskRunner taskRunner = new TaskRunner(TaskRunnerManager.this, systemConf, params);
          LOG.info("Start TaskRunner:" + taskRunner.getId());
          synchronized(taskRunnerMap) {
            taskRunnerMap.put(taskRunner.getId(), taskRunner);
          }
View Full Code Here

    StringUtils.startupShutdownMessage(TajoMaster.class, args, LOG);

    try {
      TajoMaster master = new TajoMaster();
      ShutdownHookManager.get().addShutdownHook(new CompositeServiceShutdownHook(master), SHUTDOWN_HOOK_PRIORITY);
      TajoConf conf = new TajoConf(new YarnConfiguration());
      master.init(conf);
      master.start();
    } catch (Throwable t) {
      LOG.fatal("Error starting TajoMaster", t);
      System.exit(-1);
View Full Code Here

        myServiceData, null, new HashMap<ApplicationAccessType, String>());
  }

  public static ContainerLaunchContext createCommonContainerLaunchContext(Configuration config,
                                                                          String queryId, boolean isMaster) {
    TajoConf conf = (TajoConf)config;

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    try {
      ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
      e.printStackTrace();
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the env variables to be setup
    ////////////////////////////////////////////////////////////////////////////
    LOG.info("Set the environment for the application master");

    Map<String, String> environment = new HashMap<String, String>();
    //String initialClassPath = getInitialClasspath(conf);
    environment.put(ApplicationConstants.Environment.SHELL.name(), "/bin/bash");
    if(System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()) != null) {
      environment.put(ApplicationConstants.Environment.JAVA_HOME.name(), System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()));
    }

    // TODO - to be improved with org.apache.tajo.sh shell script
    Properties prop = System.getProperties();

    if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE") ||
        (System.getenv("tajo.test") != null && System.getenv("tajo.test").equalsIgnoreCase("TRUE"))) {
      LOG.info("tajo.test is TRUE");
      environment.put(ApplicationConstants.Environment.CLASSPATH.name(), prop.getProperty("java.class.path", null));
      environment.put("tajo.test", "TRUE");
    } else {
      // Add AppMaster.jar location to classpath
      // At some point we should not be required to add
      // the hadoop specific classpaths to the env.
      // It should be provided out of the box.
      // For now setting all required classpaths including
      // the classpath to "." for the application jar
      StringBuilder classPathEnv = new StringBuilder("./");
      //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
      for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
        classPathEnv.append(':');
        classPathEnv.append(c.trim());
      }

      classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH"));
      classPathEnv.append(":./log4j.properties:./*");
      if(System.getenv("HADOOP_HOME") != null) {
        environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME"));
        environment.put(
            ApplicationConstants.Environment.HADOOP_COMMON_HOME.name(),
            System.getenv("HADOOP_HOME"));
        environment.put(
            ApplicationConstants.Environment.HADOOP_HDFS_HOME.name(),
            System.getenv("HADOOP_HOME"));
        environment.put(
            ApplicationConstants.Environment.HADOOP_YARN_HOME.name(),
            System.getenv("HADOOP_HOME"));
      }

      if(System.getenv("TAJO_BASE_CLASSPATH") != null) {
        environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH"));
      }
      environment.put(ApplicationConstants.Environment.CLASSPATH.name(), classPathEnv.toString());
    }

    ctx.setEnvironment(environment);

    if(LOG.isDebugEnabled()) {
      LOG.debug("=================================================");
      for(Map.Entry<String, String> entry: environment.entrySet()) {
        LOG.debug(entry.getKey() + "=" + entry.getValue());
      }
      LOG.debug("=================================================");
    }
    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    FileSystem fs = null;
    FileContext fsCtx = null;
    LOG.info("defaultFS: " + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY));
    try {
      fs = FileSystem.get(conf);
      fsCtx = FileContext.getFileContext(conf);
    } catch (IOException e) {
      LOG.error(e.getMessage(), e);
View Full Code Here

    options.addOption("p", "port", true, "Tajo server port");
  }

  public TajoCli(TajoConf c, String [] args,
                 InputStream in, OutputStream out) throws Exception {
    this.conf = new TajoConf(c);
    this.sin = in;
    this.reader = new ConsoleReader(sin, out);
    this.sout = new PrintWriter(reader.getOutput());

    CommandLineParser parser = new PosixParser();
View Full Code Here

  private void printInvalidCommand(String command) {
    sout.println("Invalid command " + command +". Try \\? for help.");
  }

  public static void main(String [] args) throws Exception {
    TajoConf conf = new TajoConf();
    TajoCli shell = new TajoCli(conf, args, System.in, System.out);
    System.out.println();
    int status = shell.runShell();
    System.exit(status);
  }
View Full Code Here

    assertFalse(fs.exists(tablePath));
  }

  @Test
  public final void testCreateAndDropExternalTableByExecuteQuery() throws IOException, ServiceException {
    TajoConf conf = cluster.getConfiguration();
    final String tableName = "testCreateAndDropExternalTableByExecuteQuery";

    Path tablePath = writeTmpTable(tableName);
    assertFalse(client.existTable(tableName));
View Full Code Here

    assertFalse(localFS.exists(tablePath));
  }

  @Test
  public final void testCreateAndDropTableByExecuteQuery() throws IOException, ServiceException, SQLException {
    TajoConf conf = cluster.getConfiguration();
    final String tableName = "testCreateAndDropTableByExecuteQuery";

    assertFalse(client.existTable(tableName));

    String sql = "create table " + tableName + " (deptname text, score int4)";
View Full Code Here

TOP

Related Classes of org.apache.tajo.conf.TajoConf

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.