Package org.apache.tajo.conf

Examples of org.apache.tajo.conf.TajoConf


    if (args.length < 3) {
      printUsage();
      System.exit(-1);
    }

    TajoConf conf = new TajoConf();
    Class clazz = Class.forName(args[0]);
    BenchmarkSet benchmark = (BenchmarkSet) clazz.newInstance();

    benchmark.init(conf, args[1]);
    benchmark.loadSchemas();
View Full Code Here


    for (int i = 0 ; i < sortSpecs.length; i++) {
      col = sortSpecs[i].getSortKey();
      indexKeys[i] = inSchema.getColumnId(col.getQualifiedName());
    }

    BSTIndex bst = new BSTIndex(new TajoConf());
    this.comp = new TupleComparator(keySchema, sortSpecs);
    Path storeTablePath = new Path(context.getWorkDir(), "output");
    LOG.info("Output data directory: " + storeTablePath);
    this.meta = CatalogUtil
        .newTableMeta(this.outSchema, CatalogProtos.StoreType.CSV);
View Full Code Here

  private static Random rnd = new Random(System.currentTimeMillis());

  @BeforeClass
  public static void setUp() throws Exception {
    conf = new TajoConf();
    util = new TajoTestingCluster();
    catalog = util.startCatalogCluster().getCatalog();
    workDir = CommonTestingUtil.getTestDir(TEST_PATH);
    sm = StorageManagerFactory.getStorageManager(conf, workDir);
View Full Code Here

  private TableDesc employee;

  @Before
  public void setUp() throws Exception {
    this.conf = new TajoConf();
    util = new TajoTestingCluster();
    catalog = util.startCatalogCluster().getCatalog();
    testDir = CommonTestingUtil.getTestDir(TEST_PATH);
    sm = StorageManagerFactory.getStorageManager(conf, testDir);
View Full Code Here

  private TajoTestingCluster util;

  @Before
  public void setup() throws Exception {
    this.randomValues = new HashMap<Integer, Integer>();
    this.conf = new TajoConf();
    util = new TajoTestingCluster();
    util.startCatalogCluster();
    catalog = util.getMiniCatalogCluster().getCatalog();

    Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestPhysicalPlanner");
View Full Code Here

        new HashMap<ApplicationAccessType, String>());
  }

  public static ContainerLaunchContext createCommonContainerLaunchContext(Configuration config,
                                                                          String queryId, boolean isMaster) {
    TajoConf conf = (TajoConf)config;

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

    try {
      ByteBuffer userToken = ByteBuffer.wrap(UserGroupInformation.getCurrentUser().getShortUserName().getBytes());
      ctx.setTokens(userToken);
    } catch (IOException e) {
      e.printStackTrace();
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the env variables to be setup
    ////////////////////////////////////////////////////////////////////////////
    LOG.info("Set the environment for the application master");

    Map<String, String> environment = new HashMap<String, String>();
    //String initialClassPath = getInitialClasspath(conf);
    environment.put(ApplicationConstants.Environment.SHELL.name(), "/bin/bash");
    if(System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()) != null) {
      environment.put(ApplicationConstants.Environment.JAVA_HOME.name(), System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()));
    }

    // TODO - to be improved with org.apache.tajo.sh shell script
    Properties prop = System.getProperties();

    if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE") ||
        (System.getenv("tajo.test") != null && System.getenv("tajo.test").equalsIgnoreCase("TRUE"))) {
      LOG.info("tajo.test is TRUE");
      environment.put(ApplicationConstants.Environment.CLASSPATH.name(), prop.getProperty("java.class.path", null));
      environment.put("tajo.test", "TRUE");
    } else {
      // Add AppMaster.jar location to classpath
      // At some point we should not be required to add
      // the hadoop specific classpaths to the env.
      // It should be provided out of the box.
      // For now setting all required classpaths including
      // the classpath to "." for the application jar
      StringBuilder classPathEnv = new StringBuilder("./");
      //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
      for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
        classPathEnv.append(':');
        classPathEnv.append(c.trim());
      }

      classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH"));
      classPathEnv.append(":./log4j.properties:./*");
      if(System.getenv("HADOOP_HOME") != null) {
        environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME"));
        environment.put(
            ApplicationConstants.Environment.HADOOP_COMMON_HOME.name(),
            System.getenv("HADOOP_HOME"));
        environment.put(
            ApplicationConstants.Environment.HADOOP_HDFS_HOME.name(),
            System.getenv("HADOOP_HOME"));
        environment.put(
            ApplicationConstants.Environment.HADOOP_YARN_HOME.name(),
            System.getenv("HADOOP_HOME"));
      }

      if(System.getenv("TAJO_BASE_CLASSPATH") != null) {
        environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH"));
      }
      environment.put(ApplicationConstants.Environment.CLASSPATH.name(), classPathEnv.toString());
    }

    ctx.setEnvironment(environment);

    if(LOG.isDebugEnabled()) {
      LOG.debug("=================================================");
      for(Map.Entry<String, String> entry: environment.entrySet()) {
        LOG.debug(entry.getKey() + "=" + entry.getValue());
      }
      LOG.debug("=================================================");
    }
    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LOG.info("defaultFS: " + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY));

    try {
      FileSystem fs = FileSystem.get(conf);
      FileContext fsCtx = FileContext.getFileContext(conf);
      Path systemConfPath = TajoConf.getSystemConfPath(conf);
View Full Code Here

    this.dispatcher = d.getEventHandler();
  }

  public void serviceInit(Configuration conf) throws Exception {
    Preconditions.checkArgument(conf instanceof TajoConf);
    TajoConf systemConf = (TajoConf) conf;

    // seconds
    int expireIntvl = systemConf.getIntVar(TajoConf.ConfVars.CLIENT_SESSION_EXPIRY_TIME);
    setExpireInterval(expireIntvl);
    setMonitorInterval(expireIntvl / 3);
    super.serviceInit(conf);
  }
View Full Code Here

  }

  @Override
  public void init(Configuration conf) {
    Preconditions.checkArgument(conf instanceof TajoConf);
    TajoConf tajoConf = (TajoConf) conf;
    try {
      // Setup RPC server
      InetSocketAddress initIsa =
          new InetSocketAddress("0.0.0.0", port);
      if (initIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initIsa);
      }

      int workerNum = tajoConf.getIntVar(TajoConf.ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM);
      this.rpcServer = new AsyncRpcServer(TajoWorkerProtocol.class, this, initIsa, workerNum);
      this.rpcServer.start();

      this.bindAddr = NetUtils.getConnectAddress(rpcServer.getListenAddress());
      this.addr = bindAddr.getHostName() + ":" + bindAddr.getPort();

      this.port = bindAddr.getPort();
    } catch (Exception e) {
      LOG.error(e.getMessage(), e);
    }
    // Get the master address
    LOG.info("TajoWorkerManagerService is bind to " + addr);
    tajoConf.setVar(TajoConf.ConfVars.WORKER_PEER_RPC_ADDRESS, NetUtils.normalizeInetSocketAddress(bindAddr));
    super.init(tajoConf);
  }
View Full Code Here

  public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext,
                                           int numTasks,
                                           int memoryMBPerTask) {
    int numClusterNodes = workerContext.getNumClusterNodes();

    TajoConf conf =  (TajoConf)workerContext.getQueryMaster().getConfig();
    int workerNum = conf.getIntVar(TajoConf.ConfVars.YARN_RM_WORKER_NUMBER_PER_NODE);
    return numClusterNodes == 0 ? numTasks: Math.min(numTasks, numClusterNodes * workerNum);
  }
View Full Code Here

  public void startTask(final String[] params) {
    //TODO change to use event dispatcher
    Thread t = new Thread() {
      public void run() {
        try {
          TajoConf systemConf = new TajoConf(tajoConf);
          TaskRunner taskRunner = new TaskRunner(TaskRunnerManager.this, systemConf, params);
          LOG.info("Start TaskRunner:" + taskRunner.getId());
          synchronized(taskRunnerMap) {
            taskRunnerMap.put(taskRunner.getId(), taskRunner);
          }
View Full Code Here

TOP

Related Classes of org.apache.tajo.conf.TajoConf

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.