Examples of InputConf

Examples of com.facebook.giraph.hive.impl.input.InputConf

206
207
208
209
210
211
212
213
214
215
216
217
      for (Partition partition : partitions) {
        inputInfo.addPartition(new InputPartition(table, partition));
      }
    }

    InputConf inputConf = new InputConf(conf, profileId);
    inputConf.writeNumSplitsToConf(inputDesc.getNumSplits());
    inputConf.writeProfileIdToConf();
    inputConf.writeInputInfoToConf(inputInfo);

    LOG.info("initProfile '" + profileId + "' to " + inputDesc);
  }
View Full Code Here

Examples of com.facebook.giraph.hive.impl.input.InputConf

219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
  @Override
  public List<InputSplit> getSplits(JobContext jobContext)
    throws IOException, InterruptedException {
    Configuration conf = jobContext.getConfiguration();

    InputConf inputConf = new InputConf(conf, myProfileId);

    LOG.info("getSplits for profile " + inputConf.getProfileId());

    JobConf jobConf = new JobConf(conf);
    InputInfo inputInfo = inputConf.readInputInfoFromConf();

    int partitionNum = 0;
    List<InputSplit> splits = Lists.newArrayList();
    Iterable<InputPartition> partitions = inputInfo.getPartitions();

    for (InputPartition inputPartition : partitions) {
      org.apache.hadoop.mapred.InputFormat baseInputFormat =
          inputPartition.makeInputFormat(conf);
      HadoopUtils.setInputDir(jobConf, inputPartition.getLocation());

      int splitsRequested = inputConf.readNumSplitsFromConf();
      org.apache.hadoop.mapred.InputSplit[] baseSplits =
          baseInputFormat.getSplits(jobConf, splitsRequested);
      LOG.info("Requested " + splitsRequested + " from partition (" +
          partitionNum + " out of " + Iterables.size(partitions) +
          ") values: " +
View Full Code Here

Examples of com.facebook.giraph.hive.impl.input.InputConf

200
201
202
203
204
205
206
207
208
209
210
211
      for (Partition partition : partitions) {
        inputInfo.addPartition(InputPartition.newFromHivePartition(partition));
      }
    }

    InputConf inputConf = new InputConf(conf, profileId);
    inputConf.writeNumSplitsToConf(inputDesc.getNumSplits());
    inputConf.writeProfileIdToConf();
    inputConf.writeInputInfoToConf(inputInfo);

    LOG.info("initProfile '" + profileId + "' to " + inputDesc);
  }
View Full Code Here

Examples of com.facebook.giraph.hive.impl.input.InputConf

213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
  @Override
  public List<InputSplit> getSplits(JobContext jobContext)
    throws IOException, InterruptedException {
    Configuration conf = jobContext.getConfiguration();

    InputConf inputConf = new InputConf(conf, myProfileId);

    LOG.info("getSplits for profile " + inputConf.getProfileId());

    JobConf jobConf = new JobConf(conf);
    InputInfo inputInfo = inputConf.readInputInfoFromConf();

    int partitionNum = 0;
    List<InputSplit> splits = Lists.newArrayList();
    Iterable<InputPartition> partitions = inputInfo.getPartitions();

    for (InputPartition inputPartition : partitions) {
      org.apache.hadoop.mapred.InputFormat baseInputFormat =
          inputPartition.makeInputFormat(conf);
      HadoopUtils.setInputDir(jobConf, inputPartition.getLocation());

      int splitsRequested = inputConf.readNumSplitsFromConf();
      org.apache.hadoop.mapred.InputSplit[] baseSplits =
          baseInputFormat.getSplits(jobConf, splitsRequested);
      LOG.info("Requested " + splitsRequested + " from partition (" +
          partitionNum + " out of " + Iterables.size(partitions) +
          ") values: " +
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.