Examples of InputJobInfo

org.apache.hcatalog.mapreduce.InputJobInfo
Container for metadata read from the metadata server. Prior to release 0.5, InputJobInfo was a key part of the public API, exposed directly to end-users as an argument to {@link HCatInputFormat#setInput(org.apache.hadoop.mapreduce.Job,InputJobInfo)}. Going forward, we plan on treating InputJobInfo as an implementation detail and no longer expose to end-users. Should you have a need to use InputJobInfo outside HCatalog itself, please contact the developer mailing list before depending on this class.
org.apache.hive.hcatalog.mapreduce.InputJobInfo
Container for metadata read from the metadata server. Prior to release 0.5, InputJobInfo was a key part of the public API, exposed directly to end-users as an argument to {@link HCatInputFormat#setInput(org.apache.hadoop.mapreduce.Job,InputJobInfo)}. Going forward, we plan on treating InputJobInfo as an implementation detail and no longer expose to end-users. Should you have a need to use InputJobInfo outside HCatalog itself, please contact the developer mailing list before depending on this class.

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

    // Populate jobProperties with input table name, table columns, RM snapshot,
    // hbase-default.xml and hbase-site.xml
    Map<String, String> tableJobProperties = tableDesc.getJobProperties();
    String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
    try {
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
      HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
      String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
      jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);


      Configuration jobConf = getJobConf();
      addResources(jobConf, jobProperties);
      JobConf copyOfConf = new JobConf(jobConf);
      HBaseConfiguration.addHbaseResources(copyOfConf);
      //Getting hbase delegation token in getInputSplits does not work with PIG. So need to
      //do it here
      if (jobConf instanceof JobConf) { //Should be the case
        HBaseUtil.addHBaseDelegationToken(copyOfConf);
        ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials());
      }


      String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA);
      jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema));


      String serSnapshot = (String) inputJobInfo.getProperties().get(
        HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY);
      if (serSnapshot == null) {
        HCatTableSnapshot snapshot =
          HBaseRevisionManagerUtil.createSnapshot(
            RevisionManagerConfiguration.create(copyOfConf),

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

  @Override
  public RecordReader<ImmutableBytesWritable, Result> getRecordReader(
    InputSplit split, JobConf job, Reporter reporter)
    throws IOException {
    String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO);
    InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);


    String tableName = job.get(TableInputFormat.INPUT_TABLE);
    TableSplit tSplit = (TableSplit) split;
    HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job);
    inputFormat.setConf(job);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

    //Configure projection schema
    job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema()));
    Job newJob = new Job(job);
    HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
    String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
    InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString);
    job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString);
    for (PartInfo partinfo : info.getPartitions()) {
      for (Entry<String, String> entry : partinfo.getJobProperties().entrySet())
        job.set(entry.getKey(), entry.getValue());
    }
    assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS));

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

   */
  @Override
  public ResourceStatistics getStatistics(String location, Job job) throws IOException {
    try {
      ResourceStatistics stats = new ResourceStatistics();
      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
        job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
      stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
      return stats;
    } catch (Exception e) {
      throw new IOException(e);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

    Job job = new Job(conf);
    Properties properties = new Properties();
    properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot");
    HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
    String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
    InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);


    Map<String, Long> revMap = new HashMap<String, Long>();
    revMap.put("cf1", 3L);
    revMap.put("cf2", 5L);
    TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
    HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());


    assertEquals(hcatSnapshot.getRevision("value1"), 3);
    assertEquals(hcatSnapshot.getRevision("value2"), 5);


    String dropTable = "DROP TABLE " + fullyQualTableName;
    cmdResponse = hcatDriver.run(dropTable);
    assertEquals(0, cmdResponse.getResponseCode());


    tableName = newTableName("mytableTwo");
    fullyQualTableName = databaseName + "." + tableName;
    tableQuery = "CREATE TABLE " + fullyQualTableName
      + "(key string, value1 string, value2 string) STORED BY " +
      "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'"
      + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')";
    cmdResponse = hcatDriver.run(tableQuery);
    assertEquals(0, cmdResponse.getResponseCode());
    revMap.clear();
    revMap.put("cf1", 3L);
    hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
    HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties);
    modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
    inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo);
    hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo());
    assertEquals(hcatSnapshot.getRevision("value1"), 3);
    assertEquals(hcatSnapshot.getRevision("value2"), 3);


    dropTable = "DROP TABLE " + fullyQualTableName;
    cmdResponse = hcatDriver.run(dropTable);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        if (inputJobString == null) {
            throw new IOException(
                "InputJobInfo information not found in JobContext. "
                    + "HCatInputFormat.setInput() not called?");
        }
        InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(inputJobString);
        HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil
            .convertSnapshot(snpt, inputInfo.getTableInfo());


        return hcatSnapshot;
    }

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

     */
    @Override
    public ResourceStatistics getStatistics(String location, Job job) throws IOException {
        try {
            ResourceStatistics stats = new ResourceStatistics();
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
                job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
            stats.setmBytes(getSizeInBytes(inputJobInfo) / 1024 / 1024);
            return stats;
        } catch (Exception e) {
            throw new IOException(e);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        // Populate jobProperties with input table name, table columns, RM snapshot,
        // hbase-default.xml and hbase-site.xml
        Map<String, String> tableJobProperties = tableDesc.getJobProperties();
        String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO);
        try {
            InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);
            HCatTableInfo tableInfo = inputJobInfo.getTableInfo();
            String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
            jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);


            Configuration jobConf = getJobConf();
            addResources(jobConf, jobProperties);
            JobConf copyOfConf = new JobConf(jobConf);
            HBaseConfiguration.addHbaseResources(copyOfConf);
            //Getting hbase delegation token in getInputSplits does not work with PIG. So need to
            //do it here
            if (jobConf instanceof JobConf) { //Should be the case
                HBaseUtil.addHBaseDelegationToken(copyOfConf);
                ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials());
            }


            String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA);
            jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema));


            String serSnapshot = (String) inputJobInfo.getProperties().get(
                HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY);
            if (serSnapshot == null) {
                HCatTableSnapshot snapshot =
                    HBaseRevisionManagerUtil.createSnapshot(
                        RevisionManagerConfiguration.create(copyOfConf),

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

    @Override
    public RecordReader<ImmutableBytesWritable, Result> getRecordReader(
        InputSplit split, JobConf job, Reporter reporter)
        throws IOException {
        String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO);
        InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString);


        String tableName = job.get(TableInputFormat.INPUT_TABLE);
        TableSplit tSplit = (TableSplit) split;
        HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job);
        inputFormat.setConf(job);

View Full Code Here

Examples of org.apache.hcatalog.mapreduce.InputJobInfo

        //Configure projection schema
        job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema()));
        Job newJob = new Job(job);
        HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
        InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString);
        job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString);
        for (PartInfo partinfo : info.getPartitions()) {
            for (Entry<String, String> entry : partinfo.getJobProperties().entrySet())
                job.set(entry.getKey(), entry.getValue());
        }
        assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS));

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.