Package org.apache.flink.core.memory

Examples of org.apache.flink.core.memory.InputViewDataInputStreamWrapper

600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
      final int pos = permutation1[i];
      rec.setField(pos, values[pos]);
    }
    rec.write(new OutputViewDataOutputStreamWrapper(writer));
    rec = new Record();
    rec.read(new InputViewDataInputStreamWrapper(reader));
    testAllRetrievalMethods(rec, permutation2, values);
   
    // test adding and retrieving with full stream serialization and deserialization into the same record
    rec = new Record();
    for (int i = 0; i < values.length; i++) {
      final int pos = permutation1[i];
      rec.setField(pos, values[pos]);
    }
    rec.write(new OutputViewDataOutputStreamWrapper(writer));
    rec.read(new InputViewDataInputStreamWrapper(reader));
    testAllRetrievalMethods(rec, permutation2, values);
   
    // test adding and retrieving with partial stream serialization and deserialization into a new record
    rec = new Record();
    updatePos = rnd.nextInt(values.length + 1);
    for (int i = 0; i < values.length; i++) {
      if (i == updatePos) {
        rec.write(new OutputViewDataOutputStreamWrapper(writer));
        rec = new Record();
        rec.read(new InputViewDataInputStreamWrapper(reader));
      }
     
      final int pos = permutation1[i];
      rec.setField(pos, values[pos]);
    }
    if (updatePos == values.length) {
      rec.write(new OutputViewDataOutputStreamWrapper(writer));
      rec = new Record();
      rec.read(new InputViewDataInputStreamWrapper(reader));
    }
    testAllRetrievalMethods(rec, permutation2, values);
   
    // test adding and retrieving with partial stream serialization and deserialization into the same record
    rec = new Record();
    updatePos = rnd.nextInt(values.length + 1);
    for (int i = 0; i < values.length; i++) {
      if (i == updatePos) {
        rec.write(new OutputViewDataOutputStreamWrapper(writer));
        rec.read(new InputViewDataInputStreamWrapper(reader));
      }
     
      final int pos = permutation1[i];
      rec.setField(pos, values[pos]);
    }
    if (updatePos == values.length) {
      rec.write(new OutputViewDataOutputStreamWrapper(writer));
      rec.read(new InputViewDataInputStreamWrapper(reader));
    }
    testAllRetrievalMethods(rec, permutation2, values);

    // test adding and retrieving with partial stream serialization and deserialization into a new record
    rec = new Record();
    updatePos = rnd.nextInt(values.length + 1);
    for (int i = 0; i < values.length; i++) {
      if (i == updatePos) {
        rec.write(new OutputViewDataOutputStreamWrapper(writer));
        rec = new Record();
        rec.read(new InputViewDataInputStreamWrapper(reader));
      }
     
      final int pos = permutation1[i];
      rec.setField(pos, values[pos]);
    }
    rec.write(new OutputViewDataOutputStreamWrapper(writer));
    rec = new Record();
    rec.read(new InputViewDataInputStreamWrapper(reader));
    testAllRetrievalMethods(rec, permutation2, values);
   
    // test adding and retrieving with partial stream serialization and deserialization into the same record
    rec = new Record();
    updatePos = rnd.nextInt(values.length + 1);
    for (int i = 0; i < values.length; i++) {
      if (i == updatePos) {
        rec.write(new OutputViewDataOutputStreamWrapper(writer));
        rec.read(new InputViewDataInputStreamWrapper(reader));
      }
     
      final int pos = permutation1[i];
      rec.setField(pos, values[pos]);
    }
    rec.write(new OutputViewDataOutputStreamWrapper(writer));
    rec.read(new InputViewDataInputStreamWrapper(reader));
    testAllRetrievalMethods(rec, permutation2, values);
  }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
    final DataInputStream in = new DataInputStream(bais);
   
    SimpleDistribution ddRead = new SimpleDistribution();
   
    try {
      ddRead.read(new InputViewDataInputStreamWrapper(in));
    } catch (Exception ex) {
      Assert.fail("The deserialization of the encoded data distribution caused an error");
    }
   
    Assert.assertEquals(3, ddRead.getNumberOfFields());
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
          throw new RuntimeException("User-defined accumulator class is not an Accumulator sublass.");
        }
       
        DataInputStream in = new DataInputStream(new ByteArrayInputStream(serializedData[i]));
        try {
          acc.read(new InputViewDataInputStreamWrapper(in));
          in.close();
        } catch (IOException e) {
          throw new RuntimeException("Error while deserializing the user-defined aggregate class.", e);
        }
       
View Full Code Here
869
870
871
872
873
874
875
876
877
878
879
    }

    // / Reads the connection header following version
    private void processProtocol() throws IOException {
      DataInputStream in = new DataInputStream(new ByteArrayInputStream(data.array()));
      header.read(new InputViewDataInputStreamWrapper(in));
      try {
        String protocolClassName = header.getProtocol();
        if (protocolClassName != null) {
          protocol = getProtocolClass(header.getProtocol());
        }
View Full Code Here
888
889
890
891
892
893
894
895
896
897
      DataInputStream dis = new DataInputStream(new ByteArrayInputStream(data.array()));
      int id = dis.readInt(); // try to read an id


      IOReadableWritable invocation = newInstance(invocationClass); // read param
      invocation.read(new InputViewDataInputStreamWrapper(dis));

      Call call = new Call(id, invocation, this);
      callQueue.put(call); // queue the call; maybe blocked here
    }
View Full Code Here
514
515
516
517
518
519
520
521
522
523
524
              LOG.error("Could not instantiate object of class " + c.getCanonicalName() + ".", e);
            } catch (IllegalAccessException e) {
              LOG.error("Error instantiating object of class " + c.getCanonicalName() + ".", e);
            }
            try {
              value.read(new InputViewDataInputStreamWrapper(in)); // read value
            } catch(Throwable e) {
              LOG.error("Exception while receiving an RPC call", e);
            }
          }
          call.setValue(value);
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
          throw new RuntimeException("User-defined aggregator class is not a value sublass.");
        }
       
        DataInputStream in = new DataInputStream(new ByteArrayInputStream(serializedData[i]));
        try {
          v.read(new InputViewDataInputStreamWrapper(in));
          in.close();
        } catch (IOException e) {
          throw new RuntimeException("Error while deserializing the user-defined aggregate class.", e);
        }
       
View Full Code Here
587
588
589
590
591
592
593
594
595
596
597
   
    final ByteArrayInputStream bais = new ByteArrayInputStream(stateEncoded);
    final DataInputStream in = new DataInputStream(bais);
   
    try {
      distribution.read(new InputViewDataInputStreamWrapper(in));
      return distribution;
    } catch (Exception ex) {
      throw new RuntimeException("The deserialization of the encoded data distribution state caused an error"
        + ex.getMessage() == null ? "." : ": " + ex.getMessage(), ex);
    }
View Full Code Here
217
218
219
220
221
222
223
224
225
226
227
      FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize());
      fdis.seek(file.getLen() - blockInfo.getInfoSize());

      DataInputStream input = new DataInputStream(fdis);
      blockInfo.read(new InputViewDataInputStreamWrapper(input));
      totalCount += blockInfo.getAccumulatedRecordCount();
    }

    final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount);
    return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth,
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
    this.blockInfo = this.createBlockInfo();
    if (this.splitLength > this.blockInfo.getInfoSize()) {
      // TODO: seek not supported by compressed streams. Will throw exception
      this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize());
      DataInputStream infoStream = new DataInputStream(this.stream);
      this.blockInfo.read(new InputViewDataInputStreamWrapper(infoStream));
    }

    this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart());
    this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize);
    this.dataInputStream = new DataInputStream(this.blockBasedInput);
View Full Code Here
TOP

Related Classes of org.apache.flink.core.memory.InputViewDataInputStreamWrapper

  • org.apache.flink.api.common.io.BinaryInputFormat
  • org.apache.flink.api.distributions.SimpleDataDistributionTest
  • org.apache.flink.core.testutils.CommonTestUtils
  • org.apache.flink.runtime.accumulators.AccumulatorEvent
  • org.apache.flink.runtime.io.disk.iomanager.IOManagerPerformanceBenchmark
  • org.apache.flink.runtime.ipc.Client$Connection
  • org.apache.flink.runtime.ipc.Server$Connection
  • org.apache.flink.runtime.iterative.event.EventWithAggregatorsTest
  • org.apache.flink.runtime.iterative.event.IterationEventWithAggregators
  • org.apache.flink.runtime.operators.util.OutputEmitterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.