Examples of org.apache.hcatalog.mapreduce.InputJobInfo

org.apache.hcatalog.mapreduce.InputJobInfo
Container for metadata read from the metadata server. Prior to release 0.5, InputJobInfo was a key part of the public API, exposed directly to end-users as an argument to {@link HCatInputFormat#setInput(org.apache.hadoop.mapreduce.Job,InputJobInfo)}. Going forward, we plan on treating InputJobInfo as an implementation detail and no longer expose to end-users. Should you have a need to use InputJobInfo outside HCatalog itself, please contact the developer mailing list before depending on this class.

        Job job = new Job(conf);
        Properties properties = new Properties();
        properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
        HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
        String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);


        Map<String, Long> revMap = new HashMap<String, Long>();
        revMap.put("cf1", 3L);
        revMap.put("cf2", 5L);
        TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
        HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());


        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 5);


        String dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);
        assertEquals(0, cmdResponse.getResponseCode());


        tableName = newTableName("mytableTwo");
        fullyQualTableName = databaseName + "." + tableName;
        tableQuery = "CREATE TABLE " + fullyQualTableName
            + "(key string, value1 string, value2 string) STORED BY " +
            "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
            + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')";
        cmdResponse = hcatDriver.run(tableQuery);
        assertEquals(0, cmdResponse.getResponseCode());
        revMap.clear();
        revMap.put("cf1", 3L);
        hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
        HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
        modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);
        hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());
        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 3);


        dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);

View Full Code Here

   * @throws IOException
   */
  public static void setVertexInput(Job job,
                                    InputJobInfo inputJobInfo)
    throws IOException {
    InputJobInfo vertexInputJobInfo = InputJobInfo.create(
        inputJobInfo.getDatabaseName(),
        inputJobInfo.getTableName(),
        inputJobInfo.getFilter());
    vertexInputJobInfo.getProperties().putAll(inputJobInfo.getProperties());
    Configuration conf = job.getConfiguration();
    conf.set(VERTEX_INPUT_JOB_INFO, HCatUtil.serialize(
        HCatUtils.getInputJobInfo(conf, vertexInputJobInfo)));
  }

View Full Code Here

   * @throws IOException
   */
  public static void setEdgeInput(Job job,
                                  InputJobInfo inputJobInfo)
    throws IOException {
    InputJobInfo edgeInputJobInfo = InputJobInfo.create(
        inputJobInfo.getDatabaseName(),
        inputJobInfo.getTableName(),
        inputJobInfo.getFilter());
    edgeInputJobInfo.getProperties().putAll(inputJobInfo.getProperties());
    Configuration conf = job.getConfiguration();
    conf.set(EDGE_INPUT_JOB_INFO, HCatUtil.serialize(
        HCatUtils.getInputJobInfo(conf, edgeInputJobInfo)));
  }

View Full Code Here

    GiraphJob job = new GiraphJob(getConf(), getClass().getName());
    job.getConfiguration().setVertexClass(vertexClass);


    // setup input from Hive
    if (vertexInputFormatClass != null) {
      InputJobInfo vertexInputJobInfo = InputJobInfo.create(dbName,
          vertexInputTableName, vertexInputTableFilterExpr);
      GiraphHCatInputFormat.setVertexInput(job.getInternalJob(),
          vertexInputJobInfo);
      job.getConfiguration().setVertexInputFormatClass(vertexInputFormatClass);
    }
    if (edgeInputFormatClass != null) {
      InputJobInfo edgeInputJobInfo = InputJobInfo.create(dbName,
          edgeInputTableName, edgeInputTableFilterExpr);
      GiraphHCatInputFormat.setEdgeInput(job.getInternalJob(),
          edgeInputJobInfo);
      job.getConfiguration().setEdgeInputFormatClass(edgeInputFormatClass);
    }

View Full Code Here

  @Override
  public ReaderContext prepareRead() throws HCatException {


    try {
      Job job = new Job(conf);
      InputJobInfo jobInfo = InputJobInfo.create(re.getDbName(), re.getTableName(), re.getFilterString());
      HCatInputFormat.setInput(job, jobInfo);
      HCatInputFormat hcif = new HCatInputFormat();
      ReaderContext cntxt = new ReaderContext();
      cntxt.setInputSplits(hcif.getSplits(new JobContext(job.getConfiguration(), null)));
      cntxt.setConf(job.getConfiguration());

View Full Code Here

        CommandProcessorResponse cmdResponse = hcatDriver.run(dbquery);
        assertEquals(0, cmdResponse.getResponseCode());
        cmdResponse = hcatDriver.run(tableQuery);
        assertEquals(0, cmdResponse.getResponseCode());


        InputJobInfo inputInfo = InputJobInfo.create(databaseName, tableName, null);
        Configuration conf = new Configuration(hcatConf);
        conf.set(HCatConstants.HCAT_KEY_HIVE_CONF,
                HCatUtil.serialize(getHiveConf().getAllProperties()));
        Job job = new Job(conf);
        inputInfo.getProperties().setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
        InitializeInput.setInput(job, inputInfo);
        String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);


        Map<String, Long> revMap = new HashMap<String, Long>();
        revMap.put("cf1", 3L);
        revMap.put("cf2", 5L);
        TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap,-1);
        HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());


        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 5);


        String dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);
        assertEquals(0, cmdResponse.getResponseCode());


        tableName = newTableName("mytableTwo");
        fullyQualTableName = databaseName + "." + tableName;
        tableQuery = "CREATE TABLE " + fullyQualTableName
        + "(key string, value1 string, value2 string) STORED BY " +
        "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
        + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')" ;
        cmdResponse = hcatDriver.run(tableQuery);
        assertEquals(0, cmdResponse.getResponseCode());
        revMap.clear();
        revMap.put("cf1", 3L);
        hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
        inputInfo = InputJobInfo.create(databaseName, tableName, null);
        inputInfo.getProperties().setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
        InitializeInput.setInput(job, inputInfo);
        modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);
        hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());
        assertEquals(hcatSnapshot.getRevision("value1"), 3);
        assertEquals(hcatSnapshot.getRevision("value2"), 3);


        dropTable = "DROP TABLE " + fullyQualTableName;
        cmdResponse = hcatDriver.run(dropTable);

View Full Code Here

        }
        job = new Job(conf, "hbase-bulk-aborted-transaction");
        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadAbortedTransaction.class);
        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(databaseName,
                tableName, null);
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(BytesWritable.class);

View Full Code Here

        }
        job = new Job(conf, "hbase-aborted-transaction");
        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadAbortedTransaction.class);
        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(databaseName,
                tableName, null);
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(BytesWritable.class);

View Full Code Here

        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadHTable.class);
        MapReadHTable.resetCounters();


        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(databaseName, tableName,
                null);
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(BytesWritable.class);

View Full Code Here

        // create job
        Job job = new Job(conf, "hbase-column-projection");
        job.setJarByClass(this.getClass());
        job.setMapperClass(MapReadProjHTable.class);
        job.setInputFormatClass(HCatInputFormat.class);
        InputJobInfo inputJobInfo = InputJobInfo.create(
                MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, null);
        HCatInputFormat.setOutputSchema(job, getProjectionSchema());
        HCatInputFormat.setInput(job, inputJobInfo);
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, outputDir);

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hcatalog.mapreduce.InputJobInfo

com.twitter.elephantbird.mapred.input.HiveMultiInputFormat

org.apache.giraph.io.hcatalog.GiraphHCatInputFormat

org.apache.giraph.io.hcatalog.HCatGiraphRunner

org.apache.hcatalog.data.transfer.impl.HCatInputFormatReader

org.apache.hcatalog.hbase.HBaseHCatStorageHandler

org.apache.hcatalog.hbase.HBaseInputFormat

org.apache.hcatalog.hbase.HBaseRevisionManagerUtil

org.apache.hcatalog.hbase.TestHBaseBulkOutputFormat

org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat

org.apache.hcatalog.hbase.TestHBaseInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.