Package io.druid.segment.incremental

Examples of io.druid.segment.incremental.IncrementalIndex


  }

  @Test
  public void testConcurrentAdd() throws Exception
  {
    final IncrementalIndex index = new IncrementalIndex(
        0L,
        QueryGranularity.NONE,
        new AggregatorFactory[]{new CountAggregatorFactory("count")},
        TestQueryRunners.pool
    );
    final int threadCount = 10;
    final int elementsPerThread = 200;
    final int dimensionCount = 5;
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
    final long timestamp = System.currentTimeMillis();
    final CountDownLatch latch = new CountDownLatch(threadCount);
    for (int j = 0; j < threadCount; j++) {
      executor.submit(
          new Runnable()
          {
            @Override
            public void run()
            {
              try {
                for (int i = 0; i < elementsPerThread; i++) {
                  index.add(getRow(timestamp + i, i, dimensionCount));
                }
              }
              catch (Exception e) {
                e.printStackTrace();
              }
              latch.countDown();
            }
          }
      );
    }
    Assert.assertTrue(latch.await(60, TimeUnit.SECONDS));

    Assert.assertEquals(dimensionCount, index.getDimensions().size());
    Assert.assertEquals(elementsPerThread, index.size());
    Iterator<Row> iterator = index.iterator();
    int curr = 0;
    while (iterator.hasNext()) {
      Row row = iterator.next();
      Assert.assertEquals(timestamp + curr, row.getTimestampFromEpoch());
      Assert.assertEquals(Float.valueOf(threadCount), row.getFloatMetric("count"));
View Full Code Here


  @Test
  public void testPersistCaseInsensitive() throws Exception
  {
    final long timestamp = System.currentTimeMillis();

    IncrementalIndex toPersist = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);

    final File tempDir = Files.createTempDir();
    try {
      QueryableIndex index = IndexIO.loadIndex(IndexMerger.persist(toPersist, tempDir));
View Full Code Here

  @Test
  public void testPersistMergeCaseInsensitive() throws Exception
  {
    final long timestamp = System.currentTimeMillis();
    IncrementalIndex toPersist1 = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);

    IncrementalIndex toPersist2 = new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[]{}, TestQueryRunners.pool);

    toPersist2.add(
        new MapBasedInputRow(
            timestamp,
            Arrays.asList("DIm1", "DIM2"),
            ImmutableMap.<String, Object>of("dim1", "1", "dim2", "2", "DIm1", "10000", "DIM2", "100000000")
        )
    );

    toPersist2.add(
        new MapBasedInputRow(
            timestamp,
            Arrays.asList("dIM1", "dIm2"),
            ImmutableMap.<String, Object>of("DIm1", "1", "DIM2", "2", "dim1", "5", "dim2", "6")
        )
View Full Code Here

  }

  @Test
  public void testPersistEmptyColumn() throws Exception
  {
    final IncrementalIndex toPersist1 = new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[]{}, TestQueryRunners.pool);
    final IncrementalIndex toPersist2 = new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[]{}, TestQueryRunners.pool);
    final File tmpDir1 = Files.createTempDir();
    final File tmpDir2 = Files.createTempDir();
    final File tmpDir3 = Files.createTempDir();

    try {
      toPersist1.add(
          new MapBasedInputRow(
              1L,
              ImmutableList.of("dim1", "dim2"),
              ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "foo")
          )
      );

      toPersist2.add(
          new MapBasedInputRow(
              1L,
              ImmutableList.of("dim1", "dim2"),
              ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "bar")
          )
View Full Code Here

    if (currHydrant == null) {
      throw new IAE("No currHydrant but given row[%s]", row);
    }

    synchronized (hydrantLock) {
      IncrementalIndex index = currHydrant.getIndex();
      if (index == null) {
        return -1; // the hydrant was swapped without being replaced
      }
      return index.add(row);
    }
  }
View Full Code Here

        .withMinTimestamp(minTimestamp)
        .withQueryGranularity(schema.getGranularitySpec().getQueryGranularity())
        .withDimensionsSpec(schema.getParser())
        .withMetrics(schema.getAggregators())
        .build();
    final IncrementalIndex newIndex;
    if (config.isIngestOffheap()) {
      newIndex = new OffheapIncrementalIndex(
          indexSchema,
          new OffheapBufferPool(bufferSize)
      );
    } else {
      newIndex = new IncrementalIndex(
          indexSchema,
          new OffheapBufferPool(bufferSize)
      );
    }
View Full Code Here

        new Predicate<FireHydrant>()
        {
          @Override
          public boolean apply(@Nullable FireHydrant input)
          {
            final IncrementalIndex index = input.getIndex();
            return index == null || index.size() != 0;
          }
        }
    );
  }
View Full Code Here

  public static Collection<?> makeQueryRunners(
      QueryRunnerFactory factory
  )
      throws IOException
  {
    final IncrementalIndex rtIndex = TestIndex.getIncrementalTestIndex(false);
    final QueryableIndex mMappedTestIndex = TestIndex.getMMappedTestIndex();
    final QueryableIndex mergedRealtimeIndex = TestIndex.mergedRealtimeIndex();
    final IncrementalIndex rtIndexOffheap = TestIndex.getIncrementalTestIndex(true);
    return Arrays.asList(
        new Object[][]{
            {
                makeQueryRunner(factory, new IncrementalIndexSegment(rtIndex, segmentId))
            },
View Full Code Here

  public static Collection<?> makeUnionQueryRunners(
      QueryRunnerFactory factory
  )
      throws IOException
  {
    final IncrementalIndex rtIndex = TestIndex.getIncrementalTestIndex(false);
    final QueryableIndex mMappedTestIndex = TestIndex.getMMappedTestIndex();
    final QueryableIndex mergedRealtimeIndex = TestIndex.mergedRealtimeIndex();
    final IncrementalIndex rtIndexOffheap = TestIndex.getIncrementalTestIndex(true);

    return Arrays.asList(
        new Object[][]{
            {
                makeUnionQueryRunner(factory, new IncrementalIndexSegment(rtIndex, segmentId))
View Full Code Here

          {
            return input.getOutputName();
          }
        }
    );
    final IncrementalIndex index;
    if(query.getContextValue("useOffheap", false)){
      index = new OffheapIncrementalIndex(
          // use granularity truncated min timestamp
          // since incoming truncated timestamps may precede timeStart
          granTimeStart,
          gran,
          aggs.toArray(new AggregatorFactory[aggs.size()]),
          bufferPool,
          false
      );
    } else {
     index = new IncrementalIndex(
        // use granularity truncated min timestamp
        // since incoming truncated timestamps may precede timeStart
        granTimeStart,
        gran,
        aggs.toArray(new AggregatorFactory[aggs.size()]),
View Full Code Here

TOP

Related Classes of io.druid.segment.incremental.IncrementalIndex

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.