Package io.druid.indexer

Examples of io.druid.indexer.HadoopIngestionSpec

226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
        toolbox.getConfig().getHadoopWorkingPath(),
        toolbox.getSegmentPusher().getPathForHadoop(getDataSource())
    };

    String config = (String) determineConfigurationMainMethod.invoke(null, new Object[]{determineConfigArgs});
    HadoopIngestionSpec indexerSchema = toolbox.getObjectMapper()
                                               .readValue(config, HadoopIngestionSpec.class);


    // We should have a lock from before we started running only if interval was specified
    final String version;
    if (determineIntervals) {
      Interval interval = JodaUtils.umbrellaInterval(
          JodaUtils.condenseIntervals(
              indexerSchema.getDataSchema().getGranularitySpec().bucketIntervals().get()
          )
      );
      TaskLock lock = toolbox.getTaskActionClient().submit(new LockAcquireAction(interval));
      version = lock.getVersion();
    } else {
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
    public static String runTask(String[] args) throws Exception
    {
      final String schema = args[0];
      String version = args[1];

      final HadoopIngestionSpec theSchema = HadoopDruidIndexerConfig.jsonMapper
          .readValue(
              schema,
              HadoopIngestionSpec.class
          );
      final HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSchema(
          theSchema
              .withTuningConfig(theSchema.getTuningConfig().withVersion(version))
      );

      HadoopDruidIndexerJob job = new HadoopDruidIndexerJob(
          config,
          injector.getInstance(MetadataStorageUpdaterJobHandler.class)
View Full Code Here
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
    {
      final String schema = args[0];
      final String workingPath = args[1];
      final String segmentOutputPath = args[2];

      final HadoopIngestionSpec theSchema = HadoopDruidIndexerConfig.jsonMapper
          .readValue(
              schema,
              HadoopIngestionSpec.class
          );
      final HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSchema(
          theSchema
              .withIOConfig(theSchema.getIOConfig().withSegmentOutputPath(segmentOutputPath))
              .withTuningConfig(theSchema.getTuningConfig().withWorkingPath(workingPath))
      );

      Jobby job = new HadoopDruidDetermineConfigurationJob(config);

      log.info("Starting a hadoop determine configuration job...");
View Full Code Here
346
347
348
349
350
351
352
353
354
355
356
  public void testHadoopIndexTaskSerde() throws Exception
  {
    final HadoopIndexTask task = new HadoopIndexTask(
        null,
        null,
        new HadoopIngestionSpec(
            null, null, null,
            "foo",
            new TimestampSpec("timestamp", "auto"),
            new JSONDataSpec(ImmutableList.of("foo"), null),
            new UniformGranularitySpec(
View Full Code Here
TOP

Related Classes of io.druid.indexer.HadoopIngestionSpec

  • io.druid.data.input.impl.StringInputRowParser
  • io.druid.data.input.impl.TimestampSpec
  • io.druid.indexer.partitions.PartitionsSpec
  • io.druid.indexer.partitions.SingleDimensionPartitionsSpec
  • io.druid.indexing.common.task.HadoopIndexTask
  • io.druid.indexing.common.task.HadoopIndexTask$HadoopDetermineConfigInnerProcessing
  • io.druid.indexing.common.task.HadoopIndexTask$HadoopIndexGeneratorInnerProcessing
  • io.druid.indexing.common.task.TaskSerdeTest
  • io.druid.segment.indexing.DataSchema
  • io.druid.segment.indexing.granularity.GranularitySpec
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.