Examples of org.apache.flink.types.Record

org.apache.flink.types.Record
The Record represents a multi-valued data record and forms the base of the "Record API" The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

    /**
     * Compute the new position (coordinate vector) of a cluster center.
     */
    @Override
    public void reduce(Iterator<Record> points, Collector<Record> out) {
      Record sum = sumPointsAndCount(points);
      sum.setField(1, sum.getField(1, Point.class).div(sum.getField(2, IntValue.class).getValue()));
      out.collect(sum);
    }

View Full Code Here

    public void combine(Iterator<Record> points, Collector<Record> out) {
      out.collect(sumPointsAndCount(points));
    }
    
    private final Record sumPointsAndCount(Iterator<Record> dataPoints) {
      Record next = null;
      p.clear();
      int count = 0;
      
      // compute coordinate vector sum and count
      while (dataPoints.hasNext()) {
        next = dataPoints.next();
        p.add(next.getField(1, Point.class));
        count += next.getField(2, IntValue.class).getValue();
      }
      
      next.setField(1, p);
      next.setField(2, new IntValue(count));
      return next;
    }

View Full Code Here

    double nearestDistance = Double.MAX_VALUE;
    int nearestClusterId = 0;


    // check all cluster centers
    while (pointsWithDistance.hasNext()) {
      Record res = pointsWithDistance.next();
      
      double distance = res.getField(3, DoubleValue.class).getValue();


      // compare distances
      if (distance < nearestDistance) {
        // if distance is smaller than smallest till now, update nearest cluster
        nearestDistance = distance;
        nearestClusterId = res.getField(2, IntValue.class).getValue();
        res.getFieldInto(1, this.position);
      }
    }


    // emit a new record with the center id and the data point. add a one to ease the
    // implementation of the average function with a combiner

View Full Code Here

  public static class CountAgg extends ReduceFunction {
    
    @Override
    public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {  
      long count = 0;
      Record rec = null;
      
      while(records.hasNext()) {
         rec = records.next();
         count++;
      }
      
      if(rec != null)
      {
        Tuple tuple = new Tuple();
        tuple.addAttribute("" + count);
        rec.setField(1, tuple);
      }
      
      out.collect(rec);
    }

View Full Code Here

    @Override
    public void coGroup(Iterator<Record> candidates, Iterator<Record> current, Collector<Record> out) throws Exception {
      if (!current.hasNext()) {
        throw new Exception("Error: Id not encountered before.");
      }
      Record old = current.next();
      long oldId = old.getField(1, LongValue.class).getValue();
      
      long minimumComponentID = Long.MAX_VALUE;


      while (candidates.hasNext()) {
        long candidateComponentID = candidates.next().getField(1, LongValue.class).getValue();
        if (candidateComponentID < minimumComponentID) {
          minimumComponentID = candidateComponentID;
        }
      }
      
      if (minimumComponentID < oldId) {
        newComponentId.setValue(minimumComponentID);
        old.setField(1, newComponentId);
        out.collect(old);
      }
    }

View Full Code Here

  public void combine(Iterator<Record> pointsWithDistance, Collector<Record> out) {
    double nearestDistance = Double.MAX_VALUE;


    // check all cluster centers
    while (pointsWithDistance.hasNext()) {
      Record res = pointsWithDistance.next();
      double distance = res.getField(3, DoubleValue.class).getValue();


      // compare distances
      if (distance < nearestDistance) {
        nearestDistance = distance;
        res.copyTo(this.nearest);
      }
    }


    // emit nearest one
    out.collect(this.nearest);

View Full Code Here


  private static final long serialVersionUID = 1L;


  @Override
  public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
    Record outRecord = new Record();
    Tuple returnTuple = new Tuple();
    
    long quantity = 0;
    double extendedPriceSum = 0.0; 
    
    boolean first = true;
    while(records.hasNext()) {
      Record rec = records.next();
      Tuple t = rec.getField(1, Tuple.class);
      
      if(first) {
        first = false;
        rec.copyTo(outRecord);
        returnTuple.addAttribute(rec.getField(0, StringValue.class).toString());
      }
      
      long tupleQuantity = Long.parseLong(t.getStringValueAt(4));
      quantity += tupleQuantity;

View Full Code Here

      StringTokenizer tokenizer = new StringTokenizer(line);
      while (tokenizer.hasMoreTokens()) {
        String word = tokenizer.nextToken();


        // we emit a (word, 1) pair 
        collector.collect(new Record(new StringValue(word), new IntValue(1)));
      }
    }

View Full Code Here


    private static final long serialVersionUID = 1L;


    @Override
    public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
      Record element = null;
      int sum = 0;
      
      while (records.hasNext()) {
        element = records.next();
        int cnt = element.getField(1, IntValue.class).getValue();
        sum += cnt;
      }


      element.setField(1, new IntValue(sum));
      out.collect(element);
    }

View Full Code Here

      int sum = 0;
      while (it.hasNext()) {
        sum += Integer.parseInt(it.next().getField(0, StringValue.class).getValue()) + 1;
      }
      
      out.collect(new Record(new StringValue(Integer.toString(sum))));
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.types.Record

org.apache.flink.api.common.io.DelimitedInputFormatTest

org.apache.flink.api.common.io.SequentialFormatTest

org.apache.flink.api.common.typeutils.record.RecordComparator

org.apache.flink.api.common.typeutils.record.RecordSerializer

org.apache.flink.api.java.record.CoGroupWrappingFunctionTest

org.apache.flink.api.java.record.io.CsvInputFormatTest

org.apache.flink.api.java.record.io.CsvOutputFormatTest

org.apache.flink.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

org.apache.flink.api.java.record.io.ExternalProcessInputFormatTest

org.apache.flink.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.