Examples of org.apache.accumulo.core.iterators.Combiner

Package org.apache.accumulo.core.iterators

Examples of org.apache.accumulo.core.iterators.Combiner

org.apache.accumulo.core.iterators.Combiner
A SortedKeyValueIterator that combines the Values for different versions (timestamp) of a Key within a row into a single Value. Combiner will replace one or more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method. An {@link Column}which only specifies a column family will combine all Keys in that column family individually. Similarly, a {@link Column} which specifies acolumn family and column qualifier will combine all Keys in column family and qualifier individually. Combination is only ever performed on multiple versions and not across column qualifiers or column visibilities. Implementations must provide a reduce method: {@code public Value reduce(Key key, Iterator iter)}. This reduce method will be passed the most recent Key and an iterator over the Values for all non-deleted versions of that Key. A combiner will not combine keys that differ by more than the timestamp. This class and its implementations do not automatically filter out unwanted columns from those being combined, thus it is generally recommended to use a {@link Combiner} implementation with the {@link ScannerBase#fetchColumnFamily(Text)} or {@link ScannerBase#fetchColumn(Text,Text)} methods.

    
    nkv(tm1, 1, 1, 1, 2, true, 0l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
    nkv(tm1, 1, 1, 1, 4, false, 3l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.FIXEDLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), new DefaultIteratorEnvironment());
    
    ai.seek(nr(1, 1, 1, 4, true), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 4), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2, true), ai.getTopKey());
    assertEquals("0", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    assertFalse(ai.hasTop());
    
    tm1 = new TreeMap<Key,Value>();
    nkv(tm1, 1, 1, 1, 2, true, 0l, encoder);
    ai = new SummingCombiner();
    ai.init(new SortedMapIterator(tm1), is.getOptions(), new DefaultIteratorEnvironment());
    
    ai.seek(nr(1, 1, 1, 4, true), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2, true), ai.getTopKey());
    assertEquals("0", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    assertFalse(ai.hasTop());
  }

View Full Code Here

    tm.put(new Key("r", "count", "b", 3), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 4), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 5), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "c", 6), new Value("1".getBytes()));
    SortedMapIterator smi = new SortedMapIterator(tm);
    Combiner iter = new SummingCombiner();
    IteratorSetting s = new IteratorSetting(10, "s", SummingCombiner.class);
    SummingCombiner.setColumns(s, Collections.singletonList(new IteratorSetting.Column("count")));
    SummingCombiner.setEncodingType(s, LongCombiner.StringEncoder.class);
    iter.init(smi, s.getOptions(), new DefaultIteratorEnvironment());
    Combiner iter2 = new SummingCombiner();
    IteratorSetting s2 = new IteratorSetting(10, "s2", SummingCombiner.class);
    SummingCombiner.setColumns(s2, Collections.singletonList(new IteratorSetting.Column("count","a")));
    SummingCombiner.setEncodingType(s2, LongCombiner.StringEncoder.class);
    iter2.init(iter, s.getOptions(), new DefaultIteratorEnvironment());
    iter2.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(iter2.hasTop());
    assertEquals("2", iter2.getTopValue().toString());
    iter2.next();
    assertTrue(iter2.hasTop());
    assertEquals("3", iter2.getTopValue().toString());
    iter2.next();
    assertTrue(iter2.hasTop());
    assertEquals("1", iter2.getTopValue().toString());
    iter2.next();
    assertFalse(iter2.hasTop());
  }

View Full Code Here

    // keys that aggregate
    nkv(tm1, 1, 1, 1, 1, false, 4l, encoder);
    nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 2l, encoder);
    
    Combiner ai = new MaxCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.VARLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("4", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    ai = new MinCombiner();
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
  }

View Full Code Here

    // keys that aggregate
    nkv(tm1, 1, 1, 1, 1, false, nal(1l, 2l), encoder);
    nkv(tm1, 1, 1, 1, 2, false, nal(3l, 4l, 5l), encoder);
    nkv(tm1, 1, 1, 1, 3, false, nal(), encoder);
    
    Combiner ai = new SummingArrayCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingArrayCombiner.class);
    SummingArrayCombiner.setEncodingType(is, type);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, encoderClass);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, encoderClass.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, SummingCombiner.VAR_LEN_ENCODER.getClass().getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    try {
      ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
      Assert.fail();
    } catch (IllegalArgumentException e) {}
    
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, BadEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    try {
      ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
      Assert.fail();
    } catch (IllegalArgumentException e) {}
  }

View Full Code Here

    // keys that do not aggregate
    nkv(tm1, 1, 1, 1, 1, false, 2l, encoder);
    nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.VARLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("2")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("4", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // try seeking
    
    ai.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    // seek after everything
    ai.seek(nr(1, 1, 1, 0), EMPTY_COL_FAMS, false);
    
    assertFalse(ai.hasTop());
    
  }

View Full Code Here

    nkv(tm2, 1, 1, 1, 1, false, 3l, encoder);
    
    TreeMap<Key,Value> tm3 = new TreeMap<Key,Value>();
    nkv(tm3, 1, 1, 1, 1, false, 4l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, StringEncoder.class);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    List<SortedKeyValueIterator<Key,Value>> sources = new ArrayList<SortedKeyValueIterator<Key,Value>>(3);
    sources.add(new SortedMapIterator(tm1));
    sources.add(new SortedMapIterator(tm2));
    sources.add(new SortedMapIterator(tm3));
    
    MultiIterator mi = new MultiIterator(sources, true);
    ai.init(mi, is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 1), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
  }

View Full Code Here

    // keys that aggregate
    nkv(tm1, 1, 1, 1, 1, false, 2l, encoder);
    nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, VarLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), new DefaultIteratorEnvironment());
    
    // try seeking to the beginning of a key that aggregates
    
    ai.seek(nr(1, 1, 1, 3, false), EMPTY_COL_FAMS, false);
    
    assertFalse(ai.hasTop());
    
  }

View Full Code Here

    
    nkv(tm1, 1, 1, 1, 2, true, 0l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
    nkv(tm1, 1, 1, 1, 4, false, 3l, encoder);
    
    Combiner ai = new SummingCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.FIXEDLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), new DefaultIteratorEnvironment());
    
    ai.seek(nr(1, 1, 1, 4, true), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 4), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2, true), ai.getTopKey());
    assertEquals("0", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    assertFalse(ai.hasTop());
    
    tm1 = new TreeMap<Key,Value>();
    nkv(tm1, 1, 1, 1, 2, true, 0l, encoder);
    ai = new SummingCombiner();
    ai.init(new SortedMapIterator(tm1), is.getOptions(), new DefaultIteratorEnvironment());
    
    ai.seek(nr(1, 1, 1, 4, true), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 2, true), ai.getTopKey());
    assertEquals("0", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    assertFalse(ai.hasTop());
  }

View Full Code Here

    tm.put(new Key("r", "count", "b", 3), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 4), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 5), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "c", 6), new Value("1".getBytes()));
    SortedMapIterator smi = new SortedMapIterator(tm);
    Combiner iter = new SummingCombiner();
    IteratorSetting s = new IteratorSetting(10, "s", SummingCombiner.class);
    SummingCombiner.setColumns(s, Collections.singletonList(new IteratorSetting.Column("count")));
    SummingCombiner.setEncodingType(s, LongCombiner.StringEncoder.class);
    iter.init(smi, s.getOptions(), new DefaultIteratorEnvironment());
    Combiner iter2 = new SummingCombiner();
    IteratorSetting s2 = new IteratorSetting(10, "s2", SummingCombiner.class);
    SummingCombiner.setColumns(s2, Collections.singletonList(new IteratorSetting.Column("count","a")));
    SummingCombiner.setEncodingType(s2, LongCombiner.StringEncoder.class);
    iter2.init(iter, s.getOptions(), new DefaultIteratorEnvironment());
    iter2.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(iter2.hasTop());
    assertEquals("2", iter2.getTopValue().toString());
    iter2.next();
    assertTrue(iter2.hasTop());
    assertEquals("3", iter2.getTopValue().toString());
    iter2.next();
    assertTrue(iter2.hasTop());
    assertEquals("1", iter2.getTopValue().toString());
    iter2.next();
    assertFalse(iter2.hasTop());
  }

View Full Code Here

    // keys that aggregate
    nkv(tm1, 1, 1, 1, 1, false, 4l, encoder);
    nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
    nkv(tm1, 1, 1, 1, 3, false, 2l, encoder);
    
    Combiner ai = new MaxCombiner();
    
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.VARLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("4", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
    
    ai = new MinCombiner();
    
    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    
    assertTrue(ai.hasTop());
    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    
    ai.next();
    
    assertFalse(ai.hasTop());
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.accumulo.core.iterators.Combiner

org.apache.accumulo.core.data.Key

org.apache.accumulo.core.data.Range

org.apache.accumulo.core.data.Value

org.apache.accumulo.core.iterators.conf.ColumnSet

org.apache.accumulo.core.iterators.user.BigDecimalCombinerTest

org.apache.accumulo.core.iterators.user.CombinerTest

org.apache.accumulo.examples.wikisearch.iterator.GlobalIndexUidTest

java.util.NoSuchElementException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.