Package com.facebook.giraph.hive.impl.output

Examples of com.facebook.giraph.hive.impl.output.OutputInfo

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    HiveMetaStoreClient client = new HiveMetaStoreClient(hiveConf);

    Table table = client.getTable(dbName, tableName);
    sanityCheck(table, outputDesc);

    OutputInfo oti = new OutputInfo(table);

    String partitionPiece;
    if (oti.hasPartitionInfo()) {
      partitionPiece = HiveUtils.computePartitionPath(oti.getPartitionInfo(),
          outputDesc.getPartitionValues());
    } else {
      partitionPiece = "_temp";
    }
    String partitionPath = oti.getTableRoot() + Path.SEPARATOR + partitionPiece;

    oti.setPartitionPath(partitionPath);
    HadoopUtils.setOutputDir(conf, partitionPath);

    if (oti.hasPartitionInfo()) {
      oti.setFinalOutputPath(oti.getPartitionPath());
    } else {
      oti.setFinalOutputPath(oti.getTableRoot());
    }

    HiveTableSchema tableSchema = HiveApiTableSchema.fromTable(table);
    HiveTableSchemas.putForName(conf, dbName, tableName, tableSchema);
    HiveTableSchemas.putForProfile(conf, profileId, tableSchema);
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
    throws IOException, InterruptedException {
    Configuration conf = jobContext.getConfiguration();
    OutputConf outputConf = new OutputConf(conf, myProfileId);

    HiveOutputDescription description = outputConf.readOutputDescription();
    OutputInfo oti = outputConf.readOutputTableInfo();

    if (oti.hasPartitionInfo()) {
      if (!description.hasPartitionValues()) {
        throw new IOException("table is partitioned but user input isn't");
      }
      checkPartitionDoesntExist(conf, description, oti);
    } else {
View Full Code Here
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
    HadoopUtils.setWorkOutputDir(taskAttemptContext);

    Configuration conf = taskAttemptContext.getConfiguration();
    OutputConf outputConf = new OutputConf(conf, myProfileId);

    OutputInfo oti = outputConf.readOutputTableInfo();

    HiveUtils.setRCileNumColumns(conf, oti.getColumnInfo().size());
    HadoopUtils.setOutputKeyWritableClass(conf, NullWritable.class);

    Serializer serializer = oti.createSerializer(conf);
    HadoopUtils.setOutputValueWritableClass(conf,
        serializer.getSerializedClass());

    org.apache.hadoop.mapred.OutputFormat baseOutputFormat =
        ReflectionUtils.newInstance(oti.getOutputFormatClass(), conf);
    // CHECKSTYLE: stop LineLength
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        getBaseRecordWriter(taskAttemptContext, baseOutputFormat);
    // CHECKSTYLE: resume LineLength

    StructObjectInspector soi = Inspectors.createFor(oti.getColumnInfo());

    HiveApiRecordWriter arw =
        new HiveApiRecordWriter(baseWriter, serializer, soi);

    return arw;
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
    HiveMetaStoreClient client = new HiveMetaStoreClient(hiveConf);

    Table table = client.getTable(dbName, tableName);
    sanityCheck(table, outputDesc);

    OutputInfo oti = new OutputInfo(table);

    String partitionPiece;
    if (oti.hasPartitionInfo()) {
      partitionPiece = HiveUtils.computePartitionPath(oti.getPartitionInfo(),
          outputDesc.getPartitionValues());
    } else {
      partitionPiece = "_temp";
    }
    String partitionPath = oti.getTableRoot() + Path.SEPARATOR + partitionPiece;

    oti.setPartitionPath(partitionPath);
    HadoopUtils.setOutputDir(conf, partitionPath);

    if (oti.hasPartitionInfo()) {
      oti.setFinalOutputPath(oti.getPartitionPath());
    } else {
      oti.setFinalOutputPath(oti.getTableRoot());
    }

    HiveTableSchema tableSchema = HiveApiTableSchema.fromTable(table);
    HiveTableSchemas.putForName(conf, dbName, tableName, tableSchema);
    HiveTableSchemas.putForProfile(conf, profileId, tableSchema);
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
231
    throws IOException, InterruptedException {
    Configuration conf = jobContext.getConfiguration();
    OutputConf outputConf = new OutputConf(conf, myProfileId);

    HiveOutputDescription description = outputConf.readOutputDescription();
    OutputInfo oti = outputConf.readOutputTableInfo();

    if (oti.hasPartitionInfo()) {
      if (!description.hasPartitionValues()) {
        throw new IOException("table is partitioned but user input isn't");
      }
      checkPartitionDoesntExist(conf, description, oti);
    } else {
View Full Code Here
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
    HadoopUtils.setWorkOutputDir(taskAttemptContext);

    Configuration conf = taskAttemptContext.getConfiguration();
    OutputConf outputConf = new OutputConf(conf, myProfileId);

    OutputInfo oti = outputConf.readOutputTableInfo();

    HiveUtils.setRCileNumColumns(conf, oti.getColumnInfo().size());
    HadoopUtils.setOutputKeyWritableClass(conf, NullWritable.class);

    Serializer serializer = oti.createSerializer(conf);
    HadoopUtils.setOutputValueWritableClass(conf,
        serializer.getSerializedClass());

    org.apache.hadoop.mapred.OutputFormat baseOutputFormat =
        ReflectionUtils.newInstance(oti.getOutputFormatClass(), conf);
    // CHECKSTYLE: stop LineLength
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        getBaseRecordWriter(taskAttemptContext, baseOutputFormat);
    // CHECKSTYLE: resume LineLength

    StructObjectInspector soi = Inspectors.createFor(oti.getColumnInfo());

    return new HiveApiRecordWriter(baseWriter, serializer, soi);
  }
View Full Code Here
TOP

Related Classes of com.facebook.giraph.hive.impl.output.OutputInfo

  • com.facebook.giraph.hive.output.HiveApiOutputFormat
  • org.apache.hadoop.hive.metastore.api.SerDeInfo
  • org.apache.hadoop.hive.metastore.api.StorageDescriptor
  • org.apache.hadoop.hive.serde2.Serializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.