Examples of eu.stratosphere.types.Record

eu.stratosphere.types.Record
Record is the basic data record that flows between functions in a Stratosphere program. The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

    final ChannelSelector<Record> oe1 = new RecordOutputEmitter(ShipStrategyType.PARTITION_HASH, doubleComp);


    PipedInputStream pipedInput = new PipedInputStream(1024*1024);
    DataInputStream in = new DataInputStream(pipedInput);
    DataOutputStream out;
    Record rec = null;
    
    try {
      out = new DataOutputStream(new PipedOutputStream(pipedInput));
      
      rec = new Record(1);
      rec.setField(0, new IntValue());
      
      rec.write(out);
      rec = new Record();
      rec.read(in);
    
    } catch (IOException e) {
      fail("Test erroneous");
    }

View Full Code Here

    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe = new RecordOutputEmitter(ShipStrategyType.PARTITION_RANGE, intComp, distri);
    
    final IntValue integer = new IntValue();
    final Record rec = new Record();
    
    for (int i = 0; i < NUM_ELEMENTS; i++) {
      final int nextValue = rnd.nextInt(DISTR_RANGE) + DISTR_MIN;
      integer.setValue(nextValue);
      rec.setField(0, integer);
      
      final int[] channels = oe.selectChannels(rec, NUM_BUCKETS);
      if (channels.length != 1) {
        Assert.fail("Resulting channels array has more than one channel.");
      }

View Full Code Here

      int upper = 0;
      do {
        lower = upper;
        upper = lower;


        Record target = new Record();
        // find the upper bound
        while ((target = iterator.next(target)) != null) {
          int val = target.getField(0, IntValue.class).getValue();
          Assert.assertEquals(upper++, val);
        }
        // now reset the buffer a few times
        for (int i = 0; i < 5; ++i) {
          iterator.reset();
          target = new Record();
          int count = 0;
          while ((target = iterator.next(target)) != null) {
            int val = target.getField(0, IntValue.class).getValue();
            Assert.assertEquals(lower + (count++), val);
          }
          Assert.assertEquals(upper - lower, count);
        }
      } while (iterator.nextBlock());

View Full Code Here

    private final IntValue value = new IntValue();


    @Override
    public void reduce(Iterator<Record> records, Collector<Record> out)
        throws Exception {
      Record element = null;
      int cnt = 0;
      while (records.hasNext()) {
        element = records.next();
        cnt++;
      }
      element.getField(0, this.key);
      this.value.setValue(cnt - this.key.getValue());
      element.setField(1, this.value);
      out.collect(element);
    }

View Full Code Here


    // create test objects
    final ArrayList<Record> objects = new ArrayList<Record>(NUM_TESTRECORDS);


    for (int i = 0; i < NUM_TESTRECORDS; ++i) {
      Record tmp = new Record(new IntValue(i));
      objects.add(tmp);
    }
    this.reader = new MutableObjectIteratorWrapper(objects.iterator());
  }

View Full Code Here

      // open the iterator
      iterator.open();
      
      // now test walking through the iterator
      int count = 0;
      Record target = new Record();
      while ((target = iterator.next(target)) != null) {
        Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++,
          target.getField(0, IntValue.class).getValue());
      }
      Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
      // test resetting the iterator a few times
      for (int j = 0; j < 10; ++j) {
        count = 0;
        iterator.reset();
        target = new Record();
        // now we should get the same results
        while ((target = iterator.next(target)) != null) {
          Assert.assertEquals("After reset nr. " + j + 1 + " element " + count
            + " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
        }
        Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS,
          count);
      }
      // close the iterator

View Full Code Here

    private final IntValue combineValue = new IntValue();


    @Override
    public void reduce(Iterator<Record> records, Collector<Record> out)
        throws Exception {
      Record element = null;
      int sum = 0;
      while (records.hasNext()) {
        element = records.next();
        element.getField(1, this.value);
        
        sum += this.value.getValue();
      }
      element.getField(0, this.key);
      this.value.setValue(sum - this.key.getValue());
      element.setField(1, this.value);
      out.collect(element);
    }

View Full Code Here


    // merge iterator
    MutableObjectIterator<Record> iterator = new MergeIterator<Record>(iterators, this.serializer, this.comparator);


    // check expected order
    Record rec1 = new Record();
    Record rec2 = new Record();
    final Key k1 = new Key();
    final Key k2 = new Key();
    
    int pos = 1;
    
    Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
    Assert.assertEquals(expected[0], rec1.getField(0, TestData.Key.class).getKey());
    
    while ((rec2 = iterator.next(rec2)) != null) {
      k1.setKey(rec1.getField(0, TestData.Key.class).getKey());
      k2.setKey(rec2.getField(0, TestData.Key.class).getKey());
      
      Assert.assertTrue(comparator.compare(k1, k2) <= 0);
      Assert.assertEquals(expected[pos++], k2.getKey()); 
      
      Record tmp = rec1;
      rec1 = rec2;
      rec2 = tmp;
    }
  }

View Full Code Here

    }
    
    @Override
    public void combine(Iterator<Record> records, Collector<Record> out)
        throws Exception {
      Record element = null;
      int sum = 0;
      while (records.hasNext()) {
        element = records.next();
        element.getField(1, this.combineValue);
        
        sum += this.combineValue.getValue();
      }
      
      this.combineValue.setValue(sum);
      element.setField(1, this.combineValue);
      out.collect(element);
    }

View Full Code Here

      // open the iterator
      iterator.open();


      // now test walking through the iterator
      int count = 0;
      Record target = new Record();
      while ((target = iterator.next(target)) != null) {
        Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++,
          target.getField(0, IntValue.class).getValue());
      }
      Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
      // test resetting the iterator a few times
      for (int j = 0; j < 10; ++j) {
        count = 0;
        iterator.reset();
        target = new Record();
        // now we should get the same results
        while ((target = iterator.next(target)) != null) {
          Assert.assertEquals("After reset nr. " + j + 1 + " element " + count
            + " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
        }
        Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS,
          count);
      }
      // close the iterator

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.types.Record

eu.stratosphere.api.avro.AvroWithEmptyArrayITCase$MyCoGrouper

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormatTest

eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample$ConcatenatingReducer

eu.stratosphere.api.java.record.io.CsvInputFormatTest

eu.stratosphere.api.java.record.io.CsvOutputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessInputFormatTest

eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.