Package eu.stratosphere.types

Examples of eu.stratosphere.types.Record$InternalDeSerializer


    private final IntValue theInteger = new IntValue();

    @Override
    public void reduce(Iterator<Record> records, Collector<Record> out)
        throws Exception {
      Record element = null;
      int sum = 0;
      while (records.hasNext()) {
        element = records.next();
        element.getField(1, this.theInteger);
       
        sum += this.theInteger.getValue();
      }
      this.theInteger.setValue(sum);
      element.setField(1, this.theInteger);
      out.collect(element);
    }
View Full Code Here


    private final IntValue combineValue = new IntValue();

    @Override
    public void reduce(Iterator<Record> records, Collector<Record> out)
        throws Exception {
      Record element = null;
      int sum = 0;
      while (records.hasNext()) {
        element = records.next();
        element.getField(1, this.value);
       
        sum += this.value.getValue();
      }
      element.getField(0, this.key);
      this.value.setValue(sum - this.key.getValue());
      element.setField(1, this.value);
      out.collect(element);
    }
View Full Code Here

    }
   
    @Override
    public void combine(Iterator<Record> records, Collector<Record> out)
        throws Exception {
      Record element = null;
      int sum = 0;
      while (records.hasNext()) {
        element = records.next();
        element.getField(1, this.combineValue);
       
        sum += this.combineValue.getValue();
      }
     
      if (++this.cnt >= 10) {
        throw new ExpectedTestException();
      }
     
      this.combineValue.setValue(sum);
      element.setField(1, this.combineValue);
      out.collect(element);
    }
View Full Code Here

    NormalizedKeySorter<Record> sorter = newSortBuffer(memory);
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
      ValueMode.RANDOM_LENGTH);
   
    // write the records
    Record record = new Record();
    int num = -1;
    do {
      generator.next(record);
      num++;
    }
    while (sorter.write(record));
   
    // re-read the records
    generator.reset();
    Record readTarget = new Record();
   
    int i = 0;
    while (i < num) {
      generator.next(record);
      readTarget = sorter.getRecord(readTarget, i++);
     
      Key rk = readTarget.getField(0, Key.class);
      Key gk = record.getField(0, Key.class);
     
      Value rv = readTarget.getField(1, Value.class);
      Value gv = record.getField(1, Value.class);
     
      Assert.assertEquals("The re-read key is wrong", gk, rk);
      Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
View Full Code Here

    NormalizedKeySorter<Record> sorter = newSortBuffer(memory);
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
      ValueMode.RANDOM_LENGTH);
   
    // write the records
    Record record = new Record();
    do {
      generator.next(record);
    }
    while (sorter.write(record));
   
    // re-read the records
    generator.reset();
    MutableObjectIterator<Record> iter = sorter.getIterator();
    Record readTarget = new Record();
   
    while ((readTarget = iter.next(readTarget)) != null) {
      generator.next(record);
     
      Key rk = readTarget.getField(0, Key.class);
      Key gk = record.getField(0, Key.class);
     
      Value rv = readTarget.getField(1, Value.class);
      Value gv = record.getField(1, Value.class);
     
      Assert.assertEquals("The re-read key is wrong", gk, rk);
      Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
View Full Code Here

   
    NormalizedKeySorter<Record> sorter = newSortBuffer(memory);
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
   
    // write the buffer full with the first set of records
    Record record = new Record();
    int num = -1;
    do {
      generator.next(record);
      num++;
    }
    while (sorter.write(record));
   
    sorter.reset();
   
    // write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
    generator = new TestData.Generator(SEED2, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
   
    // write the buffer full with the first set of records
    int num2 = -1;
    do {
      generator.next(record);
      num2++;
    }
    while (sorter.write(record));
   
    Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
   
    // re-read the records
    generator.reset();
    Record readTarget = new Record();
   
    int i = 0;
    while (i < num) {
      generator.next(record);
      readTarget = sorter.getRecord(readTarget, i++);
     
      Key rk = readTarget.getField(0, Key.class);
      Key gk = record.getField(0, Key.class);
     
      Value rv = readTarget.getField(1, Value.class);
      Value gv = record.getField(1, Value.class);
     
      Assert.assertEquals("The re-read key is wrong", gk, rk);
      Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
View Full Code Here

    NormalizedKeySorter<Record> sorter = newSortBuffer(memory);
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
      ValueMode.RANDOM_LENGTH);
   
    // write the records
    Record record = new Record();
    int num = -1;
    do {
      generator.next(record);
      num++;
    }
    while (sorter.write(record));
   
    // swap the records
    int start = 0, end = num - 1;
    while (start < end) {
      sorter.swap(start++, end--);
    }
   
    // re-read the records
    generator.reset();
    Record readTarget = new Record();
   
    int i = num - 1;
    while (i >= 0) {
      generator.next(record);
      readTarget = sorter.getRecord(readTarget, i--);
     
      Key rk = readTarget.getField(0, Key.class);
      Key gk = record.getField(0, Key.class);
     
      Value rv = readTarget.getField(1, Value.class);
      Value gv = record.getField(1, Value.class);
     
      Assert.assertEquals("The re-read key is wrong", gk, rk);
      Assert.assertEquals("The re-read value is wrong", gv, rv);
    }
View Full Code Here

    NormalizedKeySorter<Record> sorter = newSortBuffer(memory);
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED,
      ValueMode.RANDOM_LENGTH);
   
    // write the records
    Record record = new Record();
    int num = -1;
    do {
      generator.next(record);
      num++;
    }
View Full Code Here

    NormalizedKeySorter<Record> sorter = newSortBuffer(memory);
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
      ValueMode.RANDOM_LENGTH);
   
    // write the records
    Record record = new Record();
    int num = 0;
    do {
      generator.next(record);
      num++;
    }
    while (sorter.write(record) && num < NUM_RECORDS);
   
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
   
    MutableObjectIterator<Record> iter = sorter.getIterator();
    Record readTarget = new Record();
   
    Key current = new Key();
    Key last = new Key();
   
    iter.next(readTarget);
    readTarget.getFieldInto(0, last);
   
    while ((readTarget = iter.next(readTarget)) != null) {
      readTarget.getFieldInto(0, current);
     
      final int cmp = last.compareTo(current);
      if (cmp > 0) {
        Assert.fail("Next key is not larger or equal to previous key.");
      }
View Full Code Here

   
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, 5, KeyMode.RANDOM,
      ValueMode.FIX_LENGTH);
   
    // write the records
    Record record = new Record();
    do {
      generator.next(record);
    }
    while (sorter.write(record));
   
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
   
    MutableObjectIterator<Record> iter = sorter.getIterator();
    Record readTarget = new Record();
   
    Value current = new Value();
    Value last = new Value();
   
    iter.next(readTarget);
    readTarget.getFieldInto(1, last);
   
    while ((readTarget = iter.next(readTarget)) != null) {
      readTarget.getFieldInto(1, current);
     
      final int cmp = last.compareTo(current);
      if (cmp > 0) {
        Assert.fail("Next value is not larger or equal to previous value.");
      }
View Full Code Here

TOP

Related Classes of eu.stratosphere.types.Record$InternalDeSerializer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.