Examples of InputJobInfo

org.apache.hcatalog.mapreduce.InputJobInfo
Container for metadata read from the metadata server. Prior to release 0.5, InputJobInfo was a key part of the public API, exposed directly to end-users as an argument to {@link HCatInputFormat#setInput(org.apache.hadoop.mapreduce.Job,InputJobInfo)}. Going forward, we plan on treating InputJobInfo as an implementation detail and no longer expose to end-users. Should you have a need to use InputJobInfo outside HCatalog itself, please contact the developer mailing list before depending on this class.
org.apache.hive.hcatalog.mapreduce.InputJobInfo
Container for metadata read from the metadata server. Prior to release 0.5, InputJobInfo was a key part of the public API, exposed directly to end-users as an argument to {@link HCatInputFormat#setInput(org.apache.hadoop.mapreduce.Job,InputJobInfo)}. Going forward, we plan on treating InputJobInfo as an implementation detail and no longer expose to end-users. Should you have a need to use InputJobInfo outside HCatalog itself, please contact the developer mailing list before depending on this class.

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        Job job = new Job(conf);
        Properties properties = new Properties();
        properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
        HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
        String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);


        Map<String, Long> revMap = new HashMap<String, Long>();
        revMap.put("cf1", 3L);
        revMap.put("cf2", 5L);
        TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
        HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());


        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 5);


        String dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);
        assertEquals(0, cmdResponse.getResponseCode());


        tableName = newTableName("mytableTwo");
        fullyQualTableName = databaseName + "." + tableName;
        tableQuery = "CREATE TABLE " + fullyQualTableName
            + "(key string, value1 string, value2 string) STORED BY " +
            "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
            + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')";
        cmdResponse = hcatDriver.run(tableQuery);
        assertEquals(0, cmdResponse.getResponseCode());
        revMap.clear();
        revMap.put("cf1", 3L);
        hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
        HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
        modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);
        hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());
        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 3);


        dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

   * @throws IOException
   */
  public static void setVertexInput(Job job,
                                    InputJobInfo inputJobInfo)
    throws IOException {
    InputJobInfo vertexInputJobInfo = InputJobInfo.create(
        inputJobInfo.getDatabaseName(),
        inputJobInfo.getTableName(),
        inputJobInfo.getFilter());
    vertexInputJobInfo.getProperties().putAll(inputJobInfo.getProperties());
    Configuration conf = job.getConfiguration();
    conf.set(VERTEX_INPUT_JOB_INFO, HCatUtil.serialize(
        HCatUtils.getInputJobInfo(conf, vertexInputJobInfo)));
  }

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

   * @throws IOException
   */
  public static void setEdgeInput(Job job,
                                  InputJobInfo inputJobInfo)
    throws IOException {
    InputJobInfo edgeInputJobInfo = InputJobInfo.create(
        inputJobInfo.getDatabaseName(),
        inputJobInfo.getTableName(),
        inputJobInfo.getFilter());
    edgeInputJobInfo.getProperties().putAll(inputJobInfo.getProperties());
    Configuration conf = job.getConfiguration();
    conf.set(EDGE_INPUT_JOB_INFO, HCatUtil.serialize(
        HCatUtils.getInputJobInfo(conf, edgeInputJobInfo)));
  }

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

    GiraphJob job = new GiraphJob(getConf(), getClass().getName());
    job.getConfiguration().setVertexClass(vertexClass);


    // setup input from Hive
    if (vertexInputFormatClass != null) {
      InputJobInfo vertexInputJobInfo = InputJobInfo.create(dbName,
          vertexInputTableName, vertexInputTableFilterExpr);
      GiraphHCatInputFormat.setVertexInput(job.getInternalJob(),
          vertexInputJobInfo);
      job.getConfiguration().setVertexInputFormatClass(vertexInputFormatClass);
    }
    if (edgeInputFormatClass != null) {
      InputJobInfo edgeInputJobInfo = InputJobInfo.create(dbName,
          edgeInputTableName, edgeInputTableFilterExpr);
      GiraphHCatInputFormat.setEdgeInput(job.getInternalJob(),
          edgeInputJobInfo);
      job.getConfiguration().setEdgeInputFormatClass(edgeInputFormatClass);
    }

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

  @Override
  public ReaderContext prepareRead() throws HCatException {


    try {
      Job job = new Job(conf);
      InputJobInfo jobInfo = InputJobInfo.create(re.getDbName(), re.getTableName(), re.getFilterString());
      HCatInputFormat.setInput(job, jobInfo);
      HCatInputFormat hcif = new HCatInputFormat();
      ReaderContext cntxt = new ReaderContext();
      cntxt.setInputSplits(hcif.getSplits(new JobContext(job.getConfiguration(), null)));
      cntxt.setConf(job.getConfiguration());

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        CommandProcessorResponse cmdResponse = hcatDriver.run(dbquery);
        assertEquals(0, cmdResponse.getResponseCode());
        cmdResponse = hcatDriver.run(tableQuery);
        assertEquals(0, cmdResponse.getResponseCode());


        InputJobInfo inputInfo = InputJobInfo.create(databaseName, tableName, null);
        Configuration conf = new Configuration(hcatConf);
        conf.set(HCatConstants.HCAT_KEY_HIVE_CONF,
                HCatUtil.serialize(getHiveConf().getAllProperties()));
        Job job = new Job(conf);
        inputInfo.getProperties().setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
        InitializeInput.setInput(job, inputInfo);
        String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);


        Map<String, Long> revMap = new HashMap<String, Long>();
        revMap.put("cf1", 3L);
        revMap.put("cf2", 5L);
        TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap,-1);
        HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());


        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 5);


        String dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);
        assertEquals(0, cmdResponse.getResponseCode());


        tableName = newTableName("mytableTwo");
        fullyQualTableName = databaseName + "." + tableName;
        tableQuery = "CREATE TABLE " + fullyQualTableName
        + "(key string, value1 string, value2 string) STORED BY " +
        "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
        + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')" ;
        cmdResponse = hcatDriver.run(tableQuery);
        assertEquals(0, cmdResponse.getResponseCode());
        revMap.clear();
        revMap.put("cf1", 3L);
        hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
        inputInfo = InputJobInfo.create(databaseName, tableName, null);
        inputInfo.getProperties().setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
        InitializeInput.setInput(job, inputInfo);
        modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);
        hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());
        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 3);


        dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        }
        job = new Job(conf, "hbase-bulk-aborted-transaction");
        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadAbortedTransaction.class);
        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(databaseName,
                tableName, null);
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(BytesWritable.class);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        }
        job = new Job(conf, "hbase-aborted-transaction");
        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadAbortedTransaction.class);
        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(databaseName,
                tableName, null);
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(BytesWritable.class);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadHTable.class);
        MapReadHTable.resetCounters();


        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(databaseName, tableName,
                null);
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(BytesWritable.class);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        // create job
        Job job = new Job(conf, "hbase-column-projection");
        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadProjHTable.class);
        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(
                MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, null);
        HCatInputFormat.setOutputSchema(job, getProjectionSchema());
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.