Examples of eu.stratosphere.types.Record

eu.stratosphere.types.Record
Record is the basic data record that flows between functions in a Stratosphere program. The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

        val1Cnt++;
        records1.next();
      }
      
      while (records2.hasNext()) {
        Record record2 = records2.next();
        if (val1Cnt == 0) {
          
          if(++this.cnt>=10) {
            throw new ExpectedTestException();
          }

View Full Code Here


    public MockOutputGate(int index, List<Record> outList) {
      super(new JobID(), new GateID(), index);
      this.out = outList;
      this.deserializer = new AdaptiveSpanningRecordDeserializer<Record>();
      this.record = new Record();
    }

View Full Code Here

        join.open(buildInput, probeInput);
      } else {
        join.reopenProbe(probeInput);
      }
    
      Record record;
      final Record recordReuse = new Record();


      while (join.nextRecord())
      {
        int numBuildValues = 0;
    
        final Record probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getField(0, IntValue.class).getValue();
        
        HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
          numBuildValues = 1;
          Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());

View Full Code Here

      if(probe == 0) {
        join.open(buildInput, probeInput);
      } else {
        join.reopenProbe(probeInput);
      }
      Record record;
      final Record recordReuse = new Record();


      while (join.nextRecord())
      {  
        int numBuildValues = 0;
        
        final Record probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getField(0, IntValue.class).getValue();
        
        HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
          numBuildValues = 1;
          Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());

View Full Code Here


  private Random random;


  @Override
  public void open(Configuration parameters) {
    record = new Record();
    vertexID = new LongValue();
    partialRank = new DoubleValue();


    workerIndex = getRuntimeContext().getIndexOfThisSubtask();
    currentIteration = getIterationRuntimeContext().getSuperstepNumber();

View Full Code Here

    if (!currentPageRankIterator.hasNext()) {
      long missingVertex = partialRanks.next().getField(0, LongValue.class).getValue();
      throw new IllegalStateException("No current page rank for vertex [" + missingVertex + "]!");
    }


    Record currentPageRank = currentPageRankIterator.next();


    long edges = 0;
    double summedRank = 0;
    while (partialRanks.hasNext()) {
      summedRank += partialRanks.next().getField(1, doubleInstance).getValue();
      edges++;
    }


    double rank = BETA * summedRank + dampingFactor + danglingRankFactor;


    double currentRank = currentPageRank.getField(1, doubleInstance).getValue();
    isDangling = currentPageRank.getField(2, isDangling);


    double danglingRankToAggregate = isDangling.get() ? rank : 0;
    long danglingVerticesToAggregate = isDangling.get() ? 1 : 0;


    double diff = Math.abs(currentRank - rank);


    aggregator.aggregate(diff, rank, danglingRankToAggregate, danglingVerticesToAggregate, 1, edges, summedRank, 0);


    newRank.setValue(rank);


    accumulator.setField(0, currentPageRank.getField(0, vertexID));
    accumulator.setField(1, newRank);
    accumulator.setField(2, isDangling);


    collector.collect(accumulator);
  }

View Full Code Here

        }
        
        try {
          // ======= special-case the Record, to help the JIT and avoid some casts ======
          if (record.getClass() == Record.class) {
            Record typedRecord = (Record) record;
            @SuppressWarnings("unchecked")
            final InputFormat<Record, InputSplit> inFormat = (InputFormat<Record, InputSplit>) format;
            
            if (this.output instanceof RecordOutputCollector) {
              // Record going directly into network channels
              final RecordOutputCollector output = (RecordOutputCollector) this.output;
              while (!this.taskCanceled && !inFormat.reachedEnd()) {
                // build next pair and ship pair if it is valid
                typedRecord.clear();
                Record returnedRecord = null;
                if ((returnedRecord = inFormat.nextRecord(typedRecord)) != null) {
                  output.collect(returnedRecord);
                }
              }
            } else if (this.output instanceof ChainedCollectorMapDriver) {

View Full Code Here

   */
  
  @Override
  public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception
  {
    Record record = null;
    float amount = 0;


    while (records.hasNext()) {
      record = records.next();
      StringValue value = record.getField(1, StringValue.class);
      amount += Float.parseFloat(value.toString());
    }


    value.setValue(String.valueOf(amount));
    record.setField(1, value);
    out.collect(record);
  }

View Full Code Here

      }
    }
    this.numLeadingNormalizableKeys = nKeys;
    this.normalizableKeyPrefixLen = nKeyLen;
    
    this.temp1 = new Record();
    this.temp2 = new Record();
    
    if (sortDirection != null) {
      this.ascending = sortDirection;
    } else {
      this.ascending = new boolean[keyFields.length];

View Full Code Here

    this.normalizedKeyLengths = toCopy.normalizedKeyLengths;
    this.numLeadingNormalizableKeys = toCopy.numLeadingNormalizableKeys;
    this.normalizableKeyPrefixLen = toCopy.normalizableKeyPrefixLen;
    this.ascending = toCopy.ascending;
    
    this.temp1 = new Record();
    this.temp2 = new Record();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.types.Record

eu.stratosphere.api.avro.AvroWithEmptyArrayITCase$MyCoGrouper

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormatTest

eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample$ConcatenatingReducer

eu.stratosphere.api.java.record.io.CsvInputFormatTest

eu.stratosphere.api.java.record.io.CsvOutputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

eu.stratosphere.api.java.record.io.ExternalProcessInputFormatTest

eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.