Examples of eu.stratosphere.types.Record

eu.stratosphere.types.Record
Record is the basic data record that flows between functions in a Stratosphere program. The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

    
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
      ValueMode.FIX_LENGTH);
    
    // write the records
    Record record = new Record();
    do {
      generator.next(record);
    }
    while (sorter.write(record));
    
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
    
    MutableObjectIterator<Record> iter = sorter.getIterator();
    Record readTarget = new Record();
    
    Value current = new Value();
    Value last = new Value();
    
    iter.next(readTarget);
    readTarget.getFieldInto(1, last);
    
    while ((readTarget = iter.next(readTarget)) != null) {
      readTarget.getFieldInto(1, current);
      
      final int cmp = last.compareTo(current);
      if (cmp > 0) {
        Assert.fail("Next value is not larger or equal to previous value.");
      }

View Full Code Here

    // check order
    final MutableObjectIterator<Record> iterator = sorter.getIterator();
    int pairsEmitted = 1;
    int windowTransitions = 0;
    
    Record rec1 = new Record();
    Record rec2 = new Record();
    
    LOG.debug("Checking results...");
    Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
    while ((rec2 = iterator.next(rec2)) != null)
    {
      final TestData.Key k1 = rec1.getField(0, TestData.Key.class);
      final TestData.Key k2 = rec2.getField(0, TestData.Key.class);
      pairsEmitted++;
      
      // if the next key is smaller again, we have a new window
      if (k1.compareTo(k2) > 0) {
        windowTransitions++;
      }
      
      Record tmp = rec1;
      rec1 = rec2;
      k1.setKey(k2.getKey());
      
      rec2 = tmp;
    }

View Full Code Here

    int numRecs = 50000;
    int[] hit = new int[numChans];


    for (int i = 0; i < numRecs; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] > 0);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecs);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, stringComp);


    numChans = 100;
    numRecs = 10000;
    
    hit = new int[numChans];


    for (int i = 0; i < numRecs; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
      delegate.setInstance(rec);
        
      int[] chans = oe2.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;

View Full Code Here

    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] == (numRecords/numChannels) || hit[i] == (numRecords/numChannels)-1);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecords);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);


    numChannels = 100;
    numRecords = 10000;
    
    hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
      delegate.setInstance(rec);
        
      int[] chans = oe2.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;

View Full Code Here

    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i]+"", hit[i] == numRecords);
    }
    
    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, stringComp);


    numChannels = 100;
    numRecords = 5000;
    
    hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      StringValue k = new StringValue(i + "");
      Record rec = new Record(k);
      delegate.setInstance(rec);
        
      int[] chans = oe2.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;

View Full Code Here

    int numRecords = 5000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      Record rec = new Record(4);
      rec.setField(0, new IntValue(i));
      rec.setField(1, new StringValue("AB"+i+"CD"+i));
      rec.setField(3, new DoubleValue(i*3.141d));
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;

View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {1}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));
    delegate.setInstance(rec);
    
    try {
      oe1.selectChannels(delegate, 100);
    } catch (KeyFieldOutOfBoundsException re) {

View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    Record rec = new Record(2);
    rec.setField(1, new IntValue(1));
    delegate.setInstance(rec);


    try {
      oe1.selectChannels(delegate, 100);
    } catch (NullKeyFieldException re) {

View Full Code Here

    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    PipedInputStream pipedInput = new PipedInputStream(1024*1024);
    DataInputStream in = new DataInputStream(pipedInput);
    DataOutputStream out;
    Record rec = null;
    
    try {
      out = new DataOutputStream(new PipedOutputStream(pipedInput));
      
      rec = new Record(1);
      rec.setField(0, new IntValue());
      
      rec.write(out);
      rec = new Record();
      rec.read(in);
    
    } catch (IOException e) {
      fail("Test erroneous");
    }

View Full Code Here

      
      if (insertInvalidData) {
        bw.write("####_I_AM_INVALID_########\n");
      }
      
      Record rec = new Record();
      while ((rec = inIt.next(rec)) != null) {
        IntValue key = rec.getField(0, IntValue.class);
        IntValue value = rec.getField(1, IntValue.class);
        
        bw.write(key.getValue() + "_" + value.getValue() + "\n");
      }
      if (insertInvalidData) {
        bw.write("####_I_AM_INVALID_########\n");

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.types.Record

eu.stratosphere.api.avro.AvroWithEmptyArrayITCase$MyCoGrouper

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormatTest

eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample$ConcatenatingReducer

eu.stratosphere.api.java.record.io.CsvInputFormatTest

eu.stratosphere.api.java.record.io.CsvOutputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessInputFormatTest

eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.