Examples of org.apache.hcatalog.mapreduce.InputJobInfo

org.apache.hcatalog.mapreduce.InputJobInfo
Container for metadata read from the metadata server. Prior to release 0.5, InputJobInfo was a key part of the public API, exposed directly to end-users as an argument to {@link HCatInputFormat#setInput(org.apache.hadoop.mapreduce.Job,InputJobInfo)}. Going forward, we plan on treating InputJobInfo as an implementation detail and no longer expose to end-users. Should you have a need to use InputJobInfo outside HCatalog itself, please contact the developer mailing list before depending on this class.

    // Populate jobProperties with input table name, table columns, RM snapshot,
    // hbase-default.xml and hbase-site.xml
    Map<String, String> tableJobProperties = tableDesc.getJobProperties();
    String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
    try {
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
      HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
      String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
      jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);


      Configuration jobConf = getJobConf();
      addResources(jobConf, jobProperties);
      JobConf copyOfConf = new JobConf(jobConf);
      HBaseConfiguration.addHbaseResources(copyOfConf);
      //Getting hbase delegation token in getInputSplits does not work with PIG. So need to
      //do it here
      if (jobConf instanceof JobConf) { //Should be the case
        HBaseUtil.addHBaseDelegationToken(copyOfConf);
        ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials());
      }


      String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA);
      jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema));


      String serSnapshot = (String) inputJobInfo.getProperties().get(
        HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY);
      if (serSnapshot == null) {
        HCatTableSnapshot snapshot =
          HBaseRevisionManagerUtil.createSnapshot(
            RevisionManagerConfiguration.create(copyOfConf),

View Full Code Here

  @Override
  public RecordReader<ImmutableBytesWritable, Result> getRecordReader(
    InputSplit split, JobConf job, Reporter reporter)
    throws IOException {
    String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO);
    InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);


    String tableName = job.get(TableInputFormat.INPUT_TABLE);
    TableSplit tSplit = (TableSplit) split;
    HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job);
    inputFormat.setConf(job);

View Full Code Here

    //Configure projection schema
    job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema()));
    Job newJob = new Job(job);
    HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
    String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
    InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString);
    job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString);
    for (PartInfo partinfo : info.getPartitions()) {
      for (Entry<String, String> entry : partinfo.getJobProperties().entrySet())
        job.set(entry.getKey(), entry.getValue());
    }
    assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS));

View Full Code Here

   */
  @Override
  public ResourceStatistics getStatistics(String location, Job job) throws IOException {
    try {
      ResourceStatistics stats = new ResourceStatistics();
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
        job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
      stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
      return stats;
    } catch (Exception e) {
      throw new IOException(e);

View Full Code Here

    Job job = new Job(conf);
    Properties properties = new Properties();
    properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
    HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
    String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
    InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);


    Map<String, Long> revMap = new HashMap<String, Long>();
    revMap.put("cf1", 3L);
    revMap.put("cf2", 5L);
    TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
    HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());


    assertEquals(hcatSnapshot.getRevision("value1"), 3);
    assertEquals(hcatSnapshot.getRevision("value2"), 5);


    String dropTable = "DROP TABLE " + fullyQualTableName;
    cmdResponse = hcatDriver.run(dropTable);
    assertEquals(0, cmdResponse.getResponseCode());


    tableName = newTableName("mytableTwo");
    fullyQualTableName = databaseName + "." + tableName;
    tableQuery = "CREATE TABLE " + fullyQualTableName
      + "(key string, value1 string, value2 string) STORED BY " +
      "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
      + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')";
    cmdResponse = hcatDriver.run(tableQuery);
    assertEquals(0, cmdResponse.getResponseCode());
    revMap.clear();
    revMap.put("cf1", 3L);
    hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
    HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
    modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
    inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);
    hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());
    assertEquals(hcatSnapshot.getRevision("value1"), 3);
    assertEquals(hcatSnapshot.getRevision("value2"), 3);


    dropTable = "DROP TABLE " + fullyQualTableName;
    cmdResponse = hcatDriver.run(dropTable);

View Full Code Here

        if (inputJobString == null) {
            throw new IOException(
                "InputJobInfo information not found in JobContext. "
                    + "HCatInputFormat.setInput() not called?");
        }
        InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(inputJobString);
        HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil
            .convertSnapshot(snpt, inputInfo.getTableInfo());


        return hcatSnapshot;
    }

View Full Code Here

     */
    @Override
    public ResourceStatistics getStatistics(String location, Job job) throws IOException {
        try {
            ResourceStatistics stats = new ResourceStatistics();
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
                job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
            stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
            return stats;
        } catch (Exception e) {
            throw new IOException(e);

View Full Code Here

        // Populate jobProperties with input table name, table columns, RM snapshot,
        // hbase-default.xml and hbase-site.xml
        Map<String, String> tableJobProperties = tableDesc.getJobProperties();
        String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
        try {
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
            HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
            jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);


            Configuration jobConf = getJobConf();
            addResources(jobConf, jobProperties);
            JobConf copyOfConf = new JobConf(jobConf);
            HBaseConfiguration.addHbaseResources(copyOfConf);
            //Getting hbase delegation token in getInputSplits does not work with PIG. So need to
            //do it here
            if (jobConf instanceof JobConf) { //Should be the case
                HBaseUtil.addHBaseDelegationToken(copyOfConf);
                ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials());
            }


            String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA);
            jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema));


            String serSnapshot = (String) inputJobInfo.getProperties().get(
                HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY);
            if (serSnapshot == null) {
                HCatTableSnapshot snapshot =
                    HBaseRevisionManagerUtil.createSnapshot(
                        RevisionManagerConfiguration.create(copyOfConf),

View Full Code Here

    @Override
    public RecordReader<ImmutableBytesWritable, Result> getRecordReader(
        InputSplit split, JobConf job, Reporter reporter)
        throws IOException {
        String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO);
        InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);


        String tableName = job.get(TableInputFormat.INPUT_TABLE);
        TableSplit tSplit = (TableSplit) split;
        HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job);
        inputFormat.setConf(job);

View Full Code Here

        //Configure projection schema
        job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema()));
        Job newJob = new Job(job);
        HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString);
        job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString);
        for (PartInfo partinfo : info.getPartitions()) {
            for (Entry<String, String> entry : partinfo.getJobProperties().entrySet())
                job.set(entry.getKey(), entry.getValue());
        }
        assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS));

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hcatalog.mapreduce.InputJobInfo

com.twitter.elephantbird.mapred.input.HiveMultiInputFormat

org.apache.giraph.io.hcatalog.GiraphHCatInputFormat

org.apache.giraph.io.hcatalog.HCatGiraphRunner

org.apache.hcatalog.data.transfer.impl.HCatInputFormatReader

org.apache.hcatalog.hbase.HBaseHCatStorageHandler

org.apache.hcatalog.hbase.HBaseInputFormat

org.apache.hcatalog.hbase.HBaseRevisionManagerUtil

org.apache.hcatalog.hbase.TestHBaseBulkOutputFormat

org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat

org.apache.hcatalog.hbase.TestHBaseInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.