Examples of org.apache.flink.types.Record

org.apache.flink.types.Record
The Record represents a multi-valued data record and forms the base of the "Record API" The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

      throw new IllegalStateException("Can use either 'getOutgoingEdges()' or 'sendMessageToAllTargets()'.");
    }
    
    edgesUsed = true;
    while (edges.hasNext()) {
      Record next = edges.next();
      VertexKey k = next.getField(1, this.keyClass);
      outValue.setField(0, k);
      outValue.setField(1, m);
      out.collect(outValue);
    }
  }

View Full Code Here


  private Random random;


  @Override
  public void open(Configuration parameters) {
    record = new Record();
    vertexID = new LongValue();
    partialRank = new DoubleValue();


    workerIndex = getRuntimeContext().getIndexOfThisSubtask();
    currentIteration = getIterationRuntimeContext().getSuperstepNumber();

View Full Code Here

  
  @SuppressWarnings("unchecked")
  void init(IterationRuntimeContext context, VertexKey keyHolder, EdgeValue edgeValueHolder) {
    this.runtimeContext = context;
    this.edgeIter = new EdgesIterator<VertexKey, EdgeValue>(keyHolder, edgeValueHolder);
    this.outValue = new Record();
    this.keyClass = (Class<VertexKey>) keyHolder.getClass();
  }

View Full Code Here

      return input.hasNext();
    }


    @Override
    public Edge<VertexKey, EdgeValue> next() {
      Record next = input.next();
      next.getFieldInto(0, keyHolder);
      next.getFieldInto(1, edgeValueHolder);
      edge.set(keyHolder, edgeValueHolder);
      return edge;
    }

View Full Code Here

   */
  
  @Override
  public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception
  {
    Record record = null;
    float amount = 0;


    while (records.hasNext()) {
      record = records.next();
      StringValue value = record.getField(1, StringValue.class);
      amount += Float.parseFloat(value.toString());
    }


    value.setValue(String.valueOf(amount));
    record.setField(1, value);
    out.collect(record);
  }

View Full Code Here

    if (!currentPageRankIterator.hasNext()) {
      long missingVertex = partialRanks.next().getField(0, LongValue.class).getValue();
      throw new IllegalStateException("No current page rank for vertex [" + missingVertex + "]!");
    }


    Record currentPageRank = currentPageRankIterator.next();


    long edges = 0;
    double summedRank = 0;
    while (partialRanks.hasNext()) {
      Record pr = partialRanks.next();
      summedRank += pr.getField(1, doubleInstance).getValue();
      edges++;
    }


    double rank = BETA * summedRank + dampingFactor + danglingRankFactor;

View Full Code Here


    @Override
    public void coGroup(Iterator<Record> messages, Iterator<Record> vertex, Collector<Record> out) throws Exception {
      
      if (vertex.hasNext()) {
        Record first = vertex.next();
        first.getFieldInto(0, vertexKey);
        first.getFieldInto(1, vertexValue);
        messageIter.setSource(messages);
        vertexUpdateFunction.setOutput(first, out);
        vertexUpdateFunction.updateVertex(vertexKey, vertexValue, messageIter);
      } else {
        if (messages.hasNext()) {
          String message = "Target vertex does not exist!.";
          try {
            Record next = messages.next();
            next.getFieldInto(0, vertexKey);
            message = "Target vertex '" + vertexKey + "' does not exist!.";
          } catch (Throwable t) {}
          throw new Exception(message);
        } else {
          throw new Exception();

View Full Code Here

    private V vertexValue;
    
    @Override
    public void coGroup(Iterator<Record> edges, Iterator<Record> state, Collector<Record> out) throws Exception {
      if (state.hasNext()) {
        Record first = state.next();
        first.getFieldInto(0, vertexKey);
        first.getFieldInto(1, vertexValue);
        messagingFunction.set(edges, out);
        messagingFunction.sendMessages(vertexKey, vertexValue);
      }
    }

View Full Code Here

    if (!currentPageRankIterator.hasNext()) {
      long missingVertex = partialRanks.next().getField(0, LongValue.class).getValue();
      throw new IllegalStateException("No current page rank for vertex [" + missingVertex + "]!");
    }


    Record currentPageRank = currentPageRankIterator.next();


    long edges = 0;
    double summedRank = 0;
    while (partialRanks.hasNext()) {
      summedRank += partialRanks.next().getField(1, doubleInstance).getValue();
      edges++;
    }


    double rank = BETA * summedRank + dampingFactor + danglingRankFactor;
    double currentRank = currentPageRank.getField(1, doubleInstance).getValue();
    isDangling = currentPageRank.getField(2, isDangling);
    
    // maintain statistics to compensate for probability loss on dangling nodes
    double danglingRankToAggregate = isDangling.get() ? rank : 0;
    long danglingVerticesToAggregate = isDangling.get() ? 1 : 0;
    double diff = Math.abs(currentRank - rank);
    aggregator.aggregate(diff, rank, danglingRankToAggregate, danglingVerticesToAggregate, 1, edges);
    
    // return the new record
    newRank.setValue(rank);
    accumulator.setField(0, currentPageRank.getField(0, vertexID));
    accumulator.setField(1, newRank);
    accumulator.setField(2, isDangling);
    collector.collect(accumulator);
  }

View Full Code Here

    };
    
    int[] expectedAttrCnt = {6,2,3,3,3,5,5};


    IntTupleDataInFormat inFormat = new IntTupleDataInFormat();
    Record rec = new Record();  
    
    for(int i = 0; i < testTuples.length; i++) {
      
      byte[] tupleBytes = testTuples[i].getBytes();
      
      inFormat.readRecord(rec, tupleBytes, 0, tupleBytes.length);
      
      Assert.assertTrue("Expected Key: "+expectedKeys[i]+" != Returned Key: "+rec.getField(0, IntValue.class), rec.getField(0, IntValue.class).equals(new IntValue(expectedKeys[i])));
      Assert.assertTrue("Expected Attr Cnt: "+expectedAttrCnt[i]+" != Returned Attr Cnt: "+rec.getField(1, Tuple.class), rec.getField(1, Tuple.class).getNumberOfColumns() == expectedAttrCnt[i]);
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.types.Record

org.apache.flink.api.common.io.DelimitedInputFormatTest

org.apache.flink.api.common.io.SequentialFormatTest

org.apache.flink.api.common.typeutils.record.RecordComparator

org.apache.flink.api.common.typeutils.record.RecordSerializer

org.apache.flink.api.java.record.CoGroupWrappingFunctionTest

org.apache.flink.api.java.record.io.CsvInputFormatTest

org.apache.flink.api.java.record.io.CsvOutputFormatTest

org.apache.flink.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

org.apache.flink.api.java.record.io.ExternalProcessInputFormatTest

org.apache.flink.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.