Package org.apache.hcatalog.mapreduce

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
    // Populate RM transaction in OutputJobInfo
    // In case of bulk mode, populate intermediate output location
    Map<String, String> tableJobProperties = tableDesc.getJobProperties();
    String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
    try {
      OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString);
      HCatTableInfo tableInfo = outputJobInfo.getTableInfo();
      String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo);
      jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName);
      jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);

      Configuration jobConf = getJobConf();
      addResources(jobConf, jobProperties);

      Configuration copyOfConf = new Configuration(jobConf);
      HBaseConfiguration.addHbaseResources(copyOfConf);

      String txnString = outputJobInfo.getProperties().getProperty(
        HBaseConstants.PROPERTY_WRITE_TXN_KEY);
      Transaction txn = null;
      if (txnString == null) {
        txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo,
          RevisionManagerConfiguration.create(copyOfConf));
        String serializedTxn = HCatUtil.serialize(txn);
        outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
          serializedTxn);
      } else {
        txn = (Transaction) HCatUtil.deserialize(txnString);
      }
      if (isBulkMode(outputJobInfo)) {
        String tableLocation = tableInfo.getTableLocation();
        String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber())
          .toString();
        outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location);
        // We are writing out an intermediate sequenceFile hence
        // location is not passed in OutputJobInfo.getLocation()
        // TODO replace this with a mapreduce constant when available
        jobProperties.put("mapred.output.dir", location);
        jobProperties.put("mapred.output.committer.class", HBaseBulkOutputCommitter.class.getName());
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
    job.setOutputCommitter(HBaseBulkOutputCommitter.class);

    //manually create transaction
    RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
    try {
      OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
      Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
      outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
        HCatUtil.serialize(txn));
      job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
        HCatUtil.serialize(outputJobInfo));
    } finally {
      rm.close();
View Full Code Here
386
387
388
389
390
391
392
393
394
395
396
    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.setInputPaths(job, inputPath);


    job.setOutputFormatClass(HCatOutputFormat.class);
    OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null);
    HCatOutputFormat.setOutput(job, outputJobInfo);

    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(HCatRecord.class);
View Full Code Here
478
479
480
481
482
483
484
485
486
487
488
    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.setInputPaths(job, inputPath);


    job.setOutputFormatClass(HCatOutputFormat.class);
    OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null);
    HCatOutputFormat.setOutput(job, outputJobInfo);

    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(HCatRecord.class);
View Full Code Here
550
551
552
553
554
555
556
557
558
559
560
      os.write(Bytes.toBytes(data[i] + "\n"));
      os.close();
    }

    Path workingDir = new Path(methodTestDir, "mr_abort");
    OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
      tableName, null);
    Job job = configureJob(testName,
      conf, workingDir, MapWriteAbortTransaction.class,
      outputJobInfo, inputPath);
    assertFalse(job.waitForCompletion(true));
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
  }

  public static class MapHCatWrite extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
      HCatRecord record = new DefaultHCatRecord(3);
      HCatSchema schema = jobInfo.getOutputSchema();
      String vals[] = value.toString().split(",");
      record.setInteger("key", schema, Integer.parseInt(vals[0]));
      for (int i = 1; i < vals.length; i++) {
        String pair[] = vals[i].split(":");
        record.set(pair[0], schema, pair[1]);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
  }

  private OutputFormat<WritableComparable<?>, Put> getOutputFormat(JobConf job)
    throws IOException {
    String outputInfo = job.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
    OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(outputInfo);
    OutputFormat<WritableComparable<?>, Put> outputFormat = null;
    if (HBaseHCatStorageHandler.isBulkMode(outputJobInfo)) {
      outputFormat = new HBaseBulkOutputFormat();
    } else {
      outputFormat = new HBaseDirectOutputFormat();
View Full Code Here
404
405
406
407
408
409
410
411
412
413
414
415
416
  public static class MapHCatWrite extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {

    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
      HCatRecord record = new DefaultHCatRecord(3);
      HCatSchema schema = jobInfo.getOutputSchema();
      String vals[] = value.toString().split(",");
      record.setInteger("key", schema, Integer.parseInt(vals[0]));
      for (int i = 1; i < vals.length; i++) {
        String pair[] = vals[i].split(":");
        record.set(pair[0], schema, pair[1]);
View Full Code Here
449
450
451
452
453
454
455
456
457
458
459
460
461
    public static String failedKey;
    private static int count = 0;

    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
      HCatRecord record = new DefaultHCatRecord(3);
      HCatSchema schema = jobInfo.getOutputSchema();
      String vals[] = value.toString().split(",");
      record.setInteger("key", schema, Integer.parseInt(vals[0]));
      synchronized (MapWriteAbortTransaction.class) {
        if (count == 2) {
          failedKey = vals[0];
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
    job.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);

    //manually create transaction
    RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
    try {
      OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
      Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
      outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
        HCatUtil.serialize(txn));
      job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
        HCatUtil.serialize(outputJobInfo));
    } finally {
      rm.close();
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.mapreduce.OutputJobInfo

  • org.apache.hcatalog.data.transfer.impl.HCatOutputFormatWriter
  • org.apache.hcatalog.hbase.HBaseBaseOutputFormat
  • org.apache.hcatalog.hbase.HBaseHCatStorageHandler
  • org.apache.hcatalog.hbase.HBaseRevisionManagerUtil
  • org.apache.hcatalog.hbase.TestHBaseBulkOutputFormat
  • org.apache.hcatalog.hbase.TestHBaseBulkOutputFormat$MapHCatWrite
  • org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat
  • org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat$MapHCatWrite
  • org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat$MapWriteAbortTransaction
  • org.apache.hcatalog.mapreduce.TestHCatOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.