Examples of eu.stratosphere.types.Record

eu.stratosphere.types.Record
Record is the basic data record that flows between functions in a Stratosphere program. The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, 
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, probeInput);
    
    final Record recordReuse = new Record();
    int numRecordsInJoinResult = 0;
    
    while (join.nextRecord()) {
      HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
      while (buildSide.next(recordReuse) != null) {

View Full Code Here

        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, 
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, probeInput);
  
    Record record;
    final Record recordReuse = new Record();


    while (join.nextRecord())
    {
      int numBuildValues = 0;
      
      int key = 0;
      
      HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
      if ((record = buildSide.next(recordReuse)) != null) {
        numBuildValues = 1;
        key = record.getField(0, IntValue.class).getValue();
      }
      else {
        fail("No build side values found for a probe key.");
      }
      while ((record = buildSide.next(recordReuse)) != null) {
        numBuildValues++;
      }
      
      if (numBuildValues != 3) {
        fail("Other than 3 build values!!!");
      }
      
      Record pr = join.getCurrentProbeRecord();
      Assert.assertEquals("Probe-side key was different than build-side key.", key, pr.getField(0, IntValue.class).getValue()); 
      
      Long contained = map.get(key);
      if (contained == null) {
        contained = new Long(numBuildValues);
      }

View Full Code Here

        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, 
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, probeInput);
  
    Record record;
    final Record recordReuse = new Record();


    while (join.nextRecord())
    {  
      int numBuildValues = 0;
  
      final Record probeRec = join.getCurrentProbeRecord();
      int key = probeRec.getField(0, IntValue.class).getValue();
      
      HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
      if ((record = buildSide.next(recordReuse)) != null) {
        numBuildValues = 1;
        Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());

View Full Code Here

        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, 
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, probeInput);
    
    Record record;
    final Record recordReuse = new Record();


    while (join.nextRecord())
    {  
      int numBuildValues = 0;
      
      final Record probeRec = join.getCurrentProbeRecord();
      int key = probeRec.getField(0, IntValue.class).getValue();
      
      HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
      if ((record = buildSide.next(recordReuse)) != null) {
        numBuildValues = 1;
        Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());

View Full Code Here

        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, 
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, probeInput);
    
    final Record recordReuse = new Record();


    try {
      while (join.nextRecord()) {  
        HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
        if (buildSide.next(recordReuse) == null) {

View Full Code Here

    join.open(buildInput, new UniformRecordGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));


    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
        * NUM_PROBE_VALS;


    final Record recordReuse = new Record();
    int numRecordsInJoinResult = 0;
    
    while (join.nextRecord()) {
      HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
      while (buildSide.next(recordReuse) != null) {

View Full Code Here

        this.recordBuildSideAccesssor, this.recordProbeSideAccesssor, 
        this.recordBuildSideComparator, this.recordProbeSideComparator, this.pactRecordComparator,
        memSegments, ioManager);
    join.open(buildInput, new UniformRecordGenerator(NUM_PROBE_KEYS, NUM_PROBE_VALS, true));
    
    final Record recordReuse = new Record();
    int numRecordsInJoinResult = 0;
    
    int expectedNumResults = (Math.min(NUM_PROBE_KEYS, NUM_BUILD_KEYS) * NUM_BUILD_VALS)
    * NUM_PROBE_VALS;

View Full Code Here

  public static class Jn1 extends JoinFunction {
    private static final long serialVersionUID = 1L;


    @Override
    public void join(Record sc2, Record sc3, Collector<Record> out) throws Exception {
      Record r = new Record(3);
      r.setField(0, sc2.getField(0, StringValue.class));
      r.setField(1, sc2.getField(1, IntValue.class));
      r.setField(2, sc3.getField(1, IntValue.class));
      out.collect(r);
    }

View Full Code Here

      int c = sc1.getField(3, IntValue.class).getValue();


      int p_x = p(x, a, b, c);
      int p_y = p(y, a, b, c);
      int min = Math.min(p_x, p_y);
      out.collect(new Record(jn1.getField(0, StringValue.class), new IntValue(min)));
    }

View Full Code Here

    private static final long serialVersionUID = 1L;


    @Override
    public void map(Record jn1, Collector<Record> out) throws Exception {
      if (jn1.getField(1, IntValue.class).getValue() == jn1.getField(2, IntValue.class).getValue()) {
        out.collect(new Record(jn1.getField(0, StringValue.class), jn1.getField(1, IntValue.class)));
      }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.types.Record

eu.stratosphere.api.avro.AvroWithEmptyArrayITCase$MyCoGrouper

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormatTest

eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample$ConcatenatingReducer

eu.stratosphere.api.java.record.io.CsvInputFormatTest

eu.stratosphere.api.java.record.io.CsvOutputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessInputFormatTest

eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.