Examples of org.apache.accumulo.core.iterators.Combiner

Package org.apache.accumulo.core.iterators

Examples of org.apache.accumulo.core.iterators.Combiner

org.apache.accumulo.core.iterators.Combiner
A SortedKeyValueIterator that combines the Values for different versions (timestamp) of a Key within a row into a single Value. Combiner will replace one or more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method. An {@link Column}which only specifies a column family will combine all Keys in that column family individually. Similarly, a {@link Column} which specifies acolumn family and column qualifier will combine all Keys in column family and qualifier individually. Combination is only ever performed on multiple versions and not across column qualifiers or column visibilities. Implementations must provide a reduce method: {@code public Value reduce(Key key, Iterator iter)}. This reduce method will be passed the most recent Key and an iterator over the Values for all non-deleted versions of that Key. A combiner will not combine keys that differ by more than the timestamp. This class and its implementations do not automatically filter out unwanted columns from those being combined, thus it is generally recommended to use a {@link Combiner} implementation with the {@link ScannerBase#fetchColumnFamily(Text)} or {@link ScannerBase#fetchColumn(Text,Text)} methods.

    // keys that aggregate
    nkv(tm1, 1, 1, 1, 1, false, nal(1l, 2l), encoder);
    nkv(tm1, 1, 1, 1, 2, false, nal(3l, 4l, 5l), encoder);
    nkv(tm1, 1, 1, 1, 3, false, nal(), encoder);
    
    Combiner ai = new SummingArrayCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingArrayCombiner.class);
    SummingArrayCombiner.setEncodingType(is, type);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, encoderClass);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, encoderClass.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, SummingCombiner.VAR_LEN_ENCODER.getClass().getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    try {
      ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
      Assert.fail();
    } catch (IllegalArgumentException e) {}
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, BadEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    try {
      ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
      Assert.fail();
    } catch (IllegalArgumentException e) {}
  }

View Full Code Here

    // keys that aggregate
    nkv(tm1, 1, 1, 1, 1, false, 2l, encoder);
    nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, VarLenEncoder.class);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // try seeking to the beginning of a key that aggregates
    
    ai.seek(nr(1, 1, 1, 3), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // try seeking the middle of a key the aggregates
    ai.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    
    assertFalse(ai.hasTop());
    
    // try seeking to the end of a key the aggregates
    ai.seek(nr(1, 1, 1, 1), EMPTY_COL_FAMS, false);
    
    assertFalse(ai.hasTop());
    
    // try seeking before a key the aggregates
    ai.seek(nr(1, 1, 1, 4), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
  }

View Full Code Here

    
    // keys that do not aggregate
    nkv(tm1, 2, 2, 1, 1, false, 2l, encoder);
    nkv(tm1, 2, 2, 1, 2, false, 3l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, FixedLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // seek after key that aggregates
    ai.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    // seek before key that aggregates
    ai.seek(nr(1, 1, 1, 4), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
  }

View Full Code Here

    
    // keys that do not aggregate
    nkv(tm1, 2, 2, 1, 1, false, 2l, encoder);
    nkv(tm1, 2, 2, 1, 2, false, 3l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, FixedLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    
    SortedKeyValueIterator<Key,Value> ai2 = ai.deepCopy(null);
    SortedKeyValueIterator<Key,Value> ai3 = ai.deepCopy(null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // seek after key that aggregates
    ai2.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    
    assertTrue(ai2.hasTop());

View Full Code Here

    
    // keys that do not aggregate
    nkv(tm1, 2, 2, 1, 1, false, 2l, encoder);
    nkv(tm1, 2, 2, 1, 2, false, 3l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(0, 0, 1, 1), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // seek test
    ai.seek(nr(0, 0, 1, 0), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    // seek after key that aggregates
    ai.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    // combine all columns
    
    is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    Combiner.setCombineAllColumns(is, true);
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(0, 0, 1, 1), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
    assertEquals("5", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
  }

View Full Code Here

    // keys that do not aggregate
    nkv(tm1, 1, 1, 1, 1, false, 2l, encoder);
    nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.VARLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("2")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("4", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // try seeking
    
    ai.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // seek after everything
    ai.seek(nr(1, 1, 1, 0), EMPTY_COL_FAMS, false);
    
    assertFalse(ai.hasTop());
    
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.accumulo.core.iterators.Combiner

org.apache.accumulo.core.data.Key

org.apache.accumulo.core.data.Range

org.apache.accumulo.core.data.Value

org.apache.accumulo.core.iterators.conf.ColumnSet

org.apache.accumulo.core.iterators.user.BigDecimalCombinerTest

org.apache.accumulo.core.iterators.user.CombinerTest

org.apache.accumulo.examples.wikisearch.iterator.GlobalIndexUidTest

java.util.NoSuchElementException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.