Examples of io.druid.segment.indexing.granularity.GranularitySpec

Package io.druid.segment.indexing.granularity

Examples of io.druid.segment.indexing.granularity.GranularitySpec

io.druid.segment.indexing.granularity.GranularitySpec
Tells the indexer how to group events based on timestamp. The events may then be further partitioned based on anything, using a ShardSpec.

    }
  }


  private static Interval makeInterval(IndexIngestionSpec ingestionSchema, GranularitySpec granularitySpec)
  {
    GranularitySpec spec;
    if (ingestionSchema != null) {
      spec = ingestionSchema.getDataSchema().getGranularitySpec();
    } else {
      spec = granularitySpec;
    }


    return new Interval(
        spec.bucketIntervals().get().first().getStart(),
        spec.bucketIntervals().get().last().getEnd()
    );
  }

View Full Code Here

  }


  @Override
  public TaskStatus run(TaskToolbox toolbox) throws Exception
  {
    final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec();
    final int targetPartitionSize = ingestionSchema.getTuningConfig().getTargetPartitionSize();


    final TaskLock myLock = Iterables.getOnlyElement(getTaskLocks(toolbox));
    final Set<DataSegment> segments = Sets.newHashSet();


    final Set<Interval> validIntervals = Sets.intersection(granularitySpec.bucketIntervals().get(), getDataIntervals());
    if (validIntervals.isEmpty()) {
      throw new ISE("No valid data intervals found. Check your configs!");
    }


    for (final Interval bucket : validIntervals) {

View Full Code Here

  }


  private SortedSet<Interval> getDataIntervals() throws IOException
  {
    final FirehoseFactory firehoseFactory = ingestionSchema.getIOConfig().getFirehoseFactory();
    final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec();


    SortedSet<Interval> retVal = Sets.newTreeSet(Comparators.intervalsByStartThenEnd());
    try (Firehose firehose = firehoseFactory.connect(ingestionSchema.getDataSchema().getParser())) {
      while (firehose.hasMore()) {
        final InputRow inputRow = firehose.nextRow();
        Interval interval = granularitySpec.getSegmentGranularity()
                                           .bucket(new DateTime(inputRow.getTimestampFromEpoch()));
        retVal.add(interval);
      }
    }

View Full Code Here

          return; // we're ignoring this invalid row
        } else {
          throw e;
        }
      }
      GranularitySpec spec = config.getGranularitySpec();
      if (!spec.bucketIntervals().isPresent() || spec.bucketInterval(new DateTime(inputRow.getTimestampFromEpoch()))
                                                     .isPresent()) {
        innerMap(inputRow, value, context);
      }
    }
    catch (RuntimeException e) {

View Full Code Here

      } else {
        // Backwards compatibility
        thePartitionSpec = new SingleDimensionPartitionsSpec(partitionDimension, targetPartitionSize, null, false);
      }


      GranularitySpec theGranularitySpec = null;
      if (granularitySpec != null) {
        Preconditions.checkArgument(
            segmentGranularity == null && intervals == null,
            "Cannot mix granularitySpec with segmentGranularity/intervals"
        );
        theGranularitySpec = granularitySpec;
        if (rollupSpec != null) {
          theGranularitySpec = theGranularitySpec.withQueryGranularity(rollupSpec.rollupGranularity);
        }
      } else {
        // Backwards compatibility
        if (segmentGranularity != null && intervals != null) {
          theGranularitySpec = new UniformGranularitySpec(

View Full Code Here

TOP

Related Classes of io.druid.segment.indexing.granularity.GranularitySpec

io.druid.indexer.HadoopDruidIndexerMapper

io.druid.indexer.HadoopIngestionSpec

io.druid.indexing.common.task.IndexTask

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.