Examples of org.apache.flink.types.Record

org.apache.flink.types.Record
The Record represents a multi-valued data record and forms the base of the "Record API" The record is a tuple of arbitrary values. It implements a sparse tuple model, meaning that the record can contain many fields which are actually null and not represented in the record. It has internally a bitmap marking which fields are set and which are not.
For efficient data exchange, a record that is read from any source holds its data in serialized binary form. Fields are deserialized lazily upon first access. Modified fields are cached and the modifications are incorporated into the binary representation upon the next serialization or any explicit call to the {@link #updateBinaryRepresenation()} method.
IMPORTANT NOTE: Records must be used as mutable objects and be reused across user function calls in order to achieve performance. The record is a heavy-weight object, designed to minimize calls to the individual fields' serialization and deserialization methods. It holds quite a bit of state consumes a comparably large amount of memory (> 200 bytes in a 64 bit JVM) due to several pointers and arrays.
This class is NOT thread-safe!

    // default delimiter = '\n'
    
    format.configure(parameters);
    format.open(split);


    Record theRecord = new Record();


    assertNotNull(format.nextRecord(theRecord));
    assertEquals("my key", theRecord.getField(0, StringValue.class).getValue());
    assertEquals("my val$$$my key2", theRecord.getField(1, StringValue.class).getValue());
    
    assertNotNull(format.nextRecord(theRecord));
    assertEquals("$$ctd.$$", theRecord.getField(0, StringValue.class).getValue());
    assertEquals("my value2", theRecord.getField(1, StringValue.class).getValue());
    
    assertNull(format.nextRecord(theRecord));
    assertTrue(format.reachedEnd());
  }

View Full Code Here

  public static class Jn1 extends JoinFunction {
    private static final long serialVersionUID = 1L;


    @Override
    public void join(Record sc2, Record sc3, Collector<Record> out) throws Exception {
      Record r = new Record(3);
      r.setField(0, sc2.getField(0, StringValue.class));
      r.setField(1, sc2.getField(1, IntValue.class));
      r.setField(2, sc3.getField(1, IntValue.class));
      out.collect(r);
    }

View Full Code Here

      int c = sc1.getField(3, IntValue.class).getValue();


      int p_x = p(x, a, b, c);
      int p_y = p(y, a, b, c);
      int min = Math.min(p_x, p_y);
      out.collect(new Record(jn1.getField(0, StringValue.class), new IntValue(min)));
    }

View Full Code Here

    private static final long serialVersionUID = 1L;


    @Override
    public void map(Record jn1, Collector<Record> out) throws Exception {
      if (jn1.getField(1, IntValue.class).getValue() == jn1.getField(2, IntValue.class).getValue()) {
        out.collect(new Record(jn1.getField(0, StringValue.class), jn1.getField(1, IntValue.class)));
      }
    }

View Full Code Here

  @Test
  public void testNextKeyOnly() throws Exception
  {
    try {
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
      Assert.assertNull("KeyGroupedIterator must not have another value.", this.psi.getValues());

View Full Code Here

  {
    try {
      // Key 1, Value A
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      // Key 2, Value B
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      // Key 3, Values C, D
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
      catch (NoSuchElementException nseex) {}
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
      catch (NoSuchElementException nseex) {}
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      // Key 4, Values E, F, G
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("F"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("G"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      // Key 5, Values H, I, J, K, L
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("J"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("K"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("L"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
      catch (NoSuchElementException nseex) {}
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }

View Full Code Here

      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      
      // Progression first though haNext() and next(), then through hasNext() - Key 3, Values C, D
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      
      // Progression first via next() only, then hasNext() only Key 4, Values E, F, G
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      
      // Key 5, Values H, I, J, K, L
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      
      // end

View Full Code Here

  @Test
  public void testHasNextDoesNotOverweiteCurrentRecord() throws Exception
  {
    try {
      Iterator<Record> valsIter = null;
      Record rec = null;
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      valsIter = this.psi.getValues();
      Assert.assertNotNull("Returned Iterator must not be null", valsIter);
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));      
      Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      valsIter = this.psi.getValues();
      Assert.assertNotNull("Returned Iterator must not be null", valsIter);
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
      
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      valsIter = this.psi.getValues();
      Assert.assertNotNull("Returned Iterator must not be null", valsIter);
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail("The test encountered an unexpected exception.");
    }
  }

View Full Code Here

    FileInputSplit[] inputSplits = input.createInputSplits(0);
    Arrays.sort(inputSplits, new InputSplitSorter());
    int readCount = 0;
    for (FileInputSplit inputSplit : inputSplits) {
      input.open(inputSplit);
      Record record = new Record();
      while (!input.reachedEnd()) {
        if (input.nextRecord(record) != null) {
          this.checkEquals(this.getRecord(readCount), record);
          readCount++;
        }

View Full Code Here


  /**
   * Returns the record to write at the given position
   */
  protected Record getRecord(int index) {
    return new Record(new IntValue(index), new StringValue(String.valueOf(index)));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.types.Record

org.apache.flink.api.common.io.DelimitedInputFormatTest

org.apache.flink.api.common.io.SequentialFormatTest

org.apache.flink.api.common.typeutils.record.RecordComparator

org.apache.flink.api.common.typeutils.record.RecordSerializer

org.apache.flink.api.java.record.CoGroupWrappingFunctionTest

org.apache.flink.api.java.record.io.CsvInputFormatTest

org.apache.flink.api.java.record.io.CsvOutputFormatTest

org.apache.flink.api.java.record.io.ExternalProcessFixedLengthInputFormatTest

org.apache.flink.api.java.record.io.ExternalProcessInputFormatTest

org.apache.flink.api.java.record.io.FixedLenghtInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.