Package io.druid.timeline.partition

Examples of io.druid.timeline.partition.ShardSpec

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
      String currentPartitionStart = null;
      int currentPartitionSize = 0;
      for (final String partitionDimValue : partitionDimValues.elementSet()) {
        currentPartitionSize += partitionDimValues.count(partitionDimValue);
        if (currentPartitionSize >= targetPartitionSize) {
          final ShardSpec shardSpec = new SingleDimensionShardSpec(
              partitionDim,
              currentPartitionStart,
              partitionDimValue,
              shardSpecs.size()
          );

          log.info("Adding shard: %s", shardSpec);
          shardSpecs.add(shardSpec);

          currentPartitionSize = partitionDimValues.count(partitionDimValue);
          currentPartitionStart = partitionDimValue;
        }
      }

      if (currentPartitionSize > 0) {
        // One last shard to go
        final ShardSpec shardSpec;

        if (shardSpecs.isEmpty()) {
          shardSpec = new NoneShardSpec();
        } else {
          shardSpec = new SingleDimensionShardSpec(
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
194
195
    this.pathSpec = jsonMapper.convertValue(schema.getIOConfig().getPathSpec(), PathSpec.class);
    for (Map.Entry<DateTime, List<HadoopyShardSpec>> entry : schema.getTuningConfig().getShardSpecs().entrySet()) {
      if (entry.getValue() == null || entry.getValue().isEmpty()) {
        continue;
      }
      final ShardSpec actualSpec = entry.getValue().get(0).getActualSpec();
      shardSpecLookups.put(
          entry.getKey(), actualSpec.getLookup(
          Lists.transform(
              entry.getValue(), new Function<HadoopyShardSpec, ShardSpec>()
          {
            @Override
            public ShardSpec apply(HadoopyShardSpec input)
View Full Code Here
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    );
    if (!timeBucket.isPresent()) {
      return Optional.absent();
    }

    final ShardSpec actualSpec = shardSpecLookups.get(timeBucket.get().getStart()).getShardSpec(rollupGran.truncate(inputRow.getTimestampFromEpoch()), inputRow);
    final HadoopyShardSpec hadoopyShardSpec = hadoopShardSpecLookup.get(actualSpec);

    return Optional.of(
        new Bucket(
            hadoopyShardSpec.getShardNum(),
            timeBucket.get().getStart(),
            actualSpec.getPartitionNum()
        )
    );

  }
View Full Code Here
427
428
429
430
431
432
433
434
435
436
437
438
439
440
      VersionedIntervalTimeline<Integer, File> timeline = new VersionedIntervalTimeline<Integer, File>(
          Ordering.natural().nullsFirst()
      );

      ShardSpec noneShardSpec = new NoneShardSpec();

      for (int i = 0; i < intervals.size(); i++) {
        timeline.add(intervals.get(i), i, noneShardSpec.createChunk(filesToMap.get(i)));
      }

      List<Pair<File, Interval>> intervalsToMerge = Lists.transform(
          timeline.lookup(new Interval("1000-01-01/3000-01-01")),
          new Function<TimelineObjectHolder<Integer, File>, Pair<File, Interval>>()
View Full Code Here
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
          continue;
        }

        // See if we need to cut a new partition ending immediately before this dimension value
        if (currentDimPartition.rows > 0 && currentDimPartition.rows + dvc.numRows >= config.getTargetPartitionSize()) {
          final ShardSpec shardSpec = new SingleDimensionShardSpec(
              currentDimPartitions.dim,
              currentDimPartitionStart,
              dvc.value,
              currentDimPartitions.partitions.size()
          );

          log.info(
              "Adding possible shard with %,d rows and %,d unique values: %s",
              currentDimPartition.rows,
              currentDimPartition.cardinality,
              shardSpec
          );

          currentDimPartition.shardSpec = shardSpec;
          currentDimPartitions.partitions.add(currentDimPartition);
          currentDimPartition = new DimPartition();
          currentDimPartitionStart = dvc.value;
        }

        // Update counters
        currentDimPartition.cardinality++;
        currentDimPartition.rows += dvc.numRows;

        if (!iterator.hasNext() || !currentDimPartitions.dim.equals(iterator.peek().dim)) {
          // Finalize the current dimension

          if (currentDimPartition.rows > 0) {
            // One more shard to go
            final ShardSpec shardSpec;

            if (currentDimPartitions.partitions.isEmpty()) {
              shardSpec = new NoneShardSpec();
            } else {
              if (currentDimPartition.rows < config.getTargetPartitionSize() * SHARD_COMBINE_THRESHOLD) {
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
{
  @Test
  public void testSerdeRoundTrip() throws Exception
  {

    final ShardSpec spec = TestUtil.MAPPER.readValue(
        TestUtil.MAPPER.writeValueAsBytes(new HashBasedNumberedShardSpec(1, 2, TestUtil.MAPPER)),
        ShardSpec.class
    );
    Assert.assertEquals(1, spec.getPartitionNum());
    Assert.assertEquals(2, ((HashBasedNumberedShardSpec) spec).getPartitions());
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
  }

  @Test
  public void testSerdeBackwardsCompat() throws Exception
  {
    final ShardSpec spec = TestUtil.MAPPER.readValue(
        "{\"type\": \"hashed\", \"partitions\": 2, \"partitionNum\": 1}",
        ShardSpec.class
    );
    Assert.assertEquals(1, spec.getPartitionNum());
    Assert.assertEquals(2, ((HashBasedNumberedShardSpec) spec).getPartitions());
  }
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
public class NumberedShardSpecTest
{
  @Test
  public void testSerdeRoundTrip() throws Exception
  {
    final ShardSpec spec = TestUtil.MAPPER.readValue(
        TestUtil.MAPPER.writeValueAsBytes(new NumberedShardSpec(1, 2)),
        ShardSpec.class
    );
    Assert.assertEquals(1, spec.getPartitionNum());
    Assert.assertEquals(2, ((NumberedShardSpec) spec).getPartitions());
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
  }

  @Test
  public void testSerdeBackwardsCompat() throws Exception
  {
    final ShardSpec spec = TestUtil.MAPPER.readValue(
        "{\"type\": \"numbered\", \"partitions\": 2, \"partitionNum\": 1}",
        ShardSpec.class
    );
    Assert.assertEquals(1, spec.getPartitionNum());
    Assert.assertEquals(2, ((NumberedShardSpec) spec).getPartitions());
  }
View Full Code Here
TOP

Related Classes of io.druid.timeline.partition.ShardSpec

  • io.druid.indexer.DeterminePartitionsJob$DeterminePartitionsDimSelectionReducer
  • io.druid.indexer.HadoopDruidIndexerConfig
  • io.druid.indexing.common.task.IndexTask
  • io.druid.segment.SchemalessIndex
  • io.druid.server.shard.HashBasedNumberedShardSpecTest
  • io.druid.server.shard.NumberedShardSpecTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.