Package org.apache.hcatalog.mapreduce

Examples of org.apache.hcatalog.mapreduce.OutputJobInfo

228
229
230
231
232
233
234
235
236
237
238
      os.close();
    }

    //create job
    Path workingDir = new Path(methodTestDir, "mr_work");
    OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
      tableName, null);
    Job job = configureJob(testName, conf, workingDir, MapHCatWrite.class,
      outputJobInfo, inputPath);
    assertTrue(job.waitForCompletion(true));
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
      os.write(Bytes.toBytes(data[i] + "\n"));
      os.close();
    }

    Path workingDir = new Path(methodTestDir, "mr_abort");
    OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
      tableName, null);
    Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class,
      outputJobInfo, inputPath);
    assertFalse(job.waitForCompletion(true));
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    partitionValues.put("colname", "p1");
    //null server url means local mode
    HCatTableInfo info = HCatTableInfo.getOutputTableInfo(null, null, dbName, tblName, partitionValues);

    HCatOutputFormat.setOutput(job, info);
    OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(job);

    assertNotNull(jobInfo.getTableInfo());
    assertEquals(1, jobInfo.getTableInfo().getPartitionValues().size());
    assertEquals("p1", jobInfo.getTableInfo().getPartitionValues().get("colname"));
    assertEquals(1, jobInfo.getTableSchema().getFields().size());
    assertEquals("colname", jobInfo.getTableSchema().getFields().get(0).getName());

    StorerInfo storer = jobInfo.getStorerInfo();
    assertEquals(RCFileOutputDriver.class.getName(), storer.getOutputSDClass());

    publishTest(job);
  }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
  }

  private OutputFormat<WritableComparable<?>, Object> getOutputFormat(JobConf job)
    throws IOException {
    String outputInfo = job.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
    OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(outputInfo);
    OutputFormat<WritableComparable<?>, Object> outputFormat = null;
    if (HBaseHCatStorageHandler.isBulkMode(outputJobInfo)) {
      outputFormat = new HBaseBulkOutputFormat();
    } else {
      outputFormat = new HBaseDirectOutputFormat();
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    }

    private OutputFormat<WritableComparable<?>, Put> getOutputFormat(JobConf job)
        throws IOException {
        String outputInfo = job.get(HCatConstants.HCAT_KEY_OUTPUT_INFO);
        OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(outputInfo);
        OutputFormat<WritableComparable<?>, Put> outputFormat = null;
        if (HBaseHCatStorageHandler.isBulkMode(outputJobInfo)) {
            outputFormat = new HBaseBulkOutputFormat();
        } else {
            outputFormat = new HBaseDirectOutputFormat();
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
        job.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);

        //manually create transaction
        RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
        try {
            OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
            Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
            outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
                HCatUtil.serialize(txn));
            job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
                HCatUtil.serialize(outputJobInfo));
        } finally {
            rm.close();
View Full Code Here
228
229
230
231
232
233
234
235
236
237
238
            os.close();
        }

        //create job
        Path workingDir = new Path(methodTestDir, "mr_work");
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
            tableName, null);
        Job job = configureJob(testName, conf, workingDir, MapHCatWrite.class,
            outputJobInfo, inputPath);
        assertTrue(job.waitForCompletion(true));
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
            os.write(Bytes.toBytes(data[i] + "\n"));
            os.close();
        }

        Path workingDir = new Path(methodTestDir, "mr_abort");
        OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName,
            tableName, null);
        Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class,
            outputJobInfo, inputPath);
        assertFalse(job.waitForCompletion(true));
View Full Code Here
404
405
406
407
408
409
410
411
412
413
414
415
416
    public static class MapHCatWrite extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {

        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            for (int i = 1; i < vals.length; i++) {
                String pair[] = vals[i].split(":");
                record.set(pair[0], schema, pair[1]);
View Full Code Here
449
450
451
452
453
454
455
456
457
458
459
460
461
        public static String failedKey;
        private static int count = 0;

        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            synchronized (MapWriteAbortTransaction.class) {
                if (count == 2) {
                    failedKey = vals[0];
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.mapreduce.OutputJobInfo

  • org.apache.hcatalog.data.transfer.impl.HCatOutputFormatWriter
  • org.apache.hcatalog.hbase.HBaseBaseOutputFormat
  • org.apache.hcatalog.hbase.HBaseHCatStorageHandler
  • org.apache.hcatalog.hbase.HBaseRevisionManagerUtil
  • org.apache.hcatalog.hbase.TestHBaseBulkOutputFormat
  • org.apache.hcatalog.hbase.TestHBaseBulkOutputFormat$MapHCatWrite
  • org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat
  • org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat$MapHCatWrite
  • org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat$MapWriteAbortTransaction
  • org.apache.hcatalog.mapreduce.TestHCatOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.