Package eu.stratosphere.types

Examples of eu.stratosphere.types.Record$InternalDeSerializer

411
412
413
414
415
416
417
418
419
420
421
        val1Cnt++;
        records1.next();
      }
     
      while (records2.hasNext()) {
        Record record2 = records2.next();
        if (val1Cnt == 0) {
         
          if(++this.cnt>=10) {
            throw new ExpectedTestException();
          }
View Full Code Here
198
199
200
201
202
203
204
    public MockOutputGate(int index, List<Record> outList) {
      super(new JobID(), new GateID(), index);
      this.out = outList;
      this.deserializer = new AdaptiveSpanningRecordDeserializer<Record>();
      this.record = new Record();
    }
View Full Code Here
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
        join.open(buildInput, probeInput);
      } else {
        join.reopenProbe(probeInput);
      }
   
      Record record;
      final Record recordReuse = new Record();

      while (join.nextRecord())
      {
        int numBuildValues = 0;
   
        final Record probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getField(0, IntValue.class).getValue();
       
        HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
          numBuildValues = 1;
          Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());
View Full Code Here
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
      if(probe == 0) {
        join.open(buildInput, probeInput);
      } else {
        join.reopenProbe(probeInput);
      }
      Record record;
      final Record recordReuse = new Record();

      while (join.nextRecord())
      { 
        int numBuildValues = 0;
       
        final Record probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getField(0, IntValue.class).getValue();
       
        HashBucketIterator<Record, Record> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
          numBuildValues = 1;
          Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getField(0, IntValue.class).getValue());
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
  private Random random;

  @Override
  public void open(Configuration parameters) {
    record = new Record();
    vertexID = new LongValue();
    partialRank = new DoubleValue();

    workerIndex = getRuntimeContext().getIndexOfThisSubtask();
    currentIteration = getIterationRuntimeContext().getSuperstepNumber();
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    if (!currentPageRankIterator.hasNext()) {
      long missingVertex = partialRanks.next().getField(0, LongValue.class).getValue();
      throw new IllegalStateException("No current page rank for vertex [" + missingVertex + "]!");
    }

    Record currentPageRank = currentPageRankIterator.next();

    long edges = 0;
    double summedRank = 0;
    while (partialRanks.hasNext()) {
      summedRank += partialRanks.next().getField(1, doubleInstance).getValue();
      edges++;
    }

    double rank = BETA * summedRank + dampingFactor + danglingRankFactor;

    double currentRank = currentPageRank.getField(1, doubleInstance).getValue();
    isDangling = currentPageRank.getField(2, isDangling);

    double danglingRankToAggregate = isDangling.get() ? rank : 0;
    long danglingVerticesToAggregate = isDangling.get() ? 1 : 0;

    double diff = Math.abs(currentRank - rank);

    aggregator.aggregate(diff, rank, danglingRankToAggregate, danglingVerticesToAggregate, 1, edges, summedRank, 0);

    newRank.setValue(rank);

    accumulator.setField(0, currentPageRank.getField(0, vertexID));
    accumulator.setField(1, newRank);
    accumulator.setField(2, isDangling);

    collector.collect(accumulator);
  }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
        }
       
        try {
          // ======= special-case the Record, to help the JIT and avoid some casts ======
          if (record.getClass() == Record.class) {
            Record typedRecord = (Record) record;
            @SuppressWarnings("unchecked")
            final InputFormat<Record, InputSplit> inFormat = (InputFormat<Record, InputSplit>) format;
           
            if (this.output instanceof RecordOutputCollector) {
              // Record going directly into network channels
              final RecordOutputCollector output = (RecordOutputCollector) this.output;
              while (!this.taskCanceled && !inFormat.reachedEnd()) {
                // build next pair and ship pair if it is valid
                typedRecord.clear();
                Record returnedRecord = null;
                if ((returnedRecord = inFormat.nextRecord(typedRecord)) != null) {
                  output.collect(returnedRecord);
                }
              }
            } else if (this.output instanceof ChainedCollectorMapDriver) {
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
   */
 
  @Override
  public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception
  {
    Record record = null;
    float amount = 0;

    while (records.hasNext()) {
      record = records.next();
      StringValue value = record.getField(1, StringValue.class);
      amount += Float.parseFloat(value.toString());
    }

    value.setValue(String.valueOf(amount));
    record.setField(1, value);
    out.collect(record);
  }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
      }
    }
    this.numLeadingNormalizableKeys = nKeys;
    this.normalizableKeyPrefixLen = nKeyLen;
   
    this.temp1 = new Record();
    this.temp2 = new Record();
   
    if (sortDirection != null) {
      this.ascending = sortDirection;
    } else {
      this.ascending = new boolean[keyFields.length];
View Full Code Here
174
175
176
177
178
179
180
181
    this.normalizedKeyLengths = toCopy.normalizedKeyLengths;
    this.numLeadingNormalizableKeys = toCopy.numLeadingNormalizableKeys;
    this.normalizableKeyPrefixLen = toCopy.normalizableKeyPrefixLen;
    this.ascending = toCopy.ascending;
   
    this.temp1 = new Record();
    this.temp2 = new Record();
  }
View Full Code Here
TOP

Related Classes of eu.stratosphere.types.Record$InternalDeSerializer

  • eu.stratosphere.api.avro.AvroWithEmptyArrayITCase$MyCoGrouper
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.SequentialFormatTest
  • eu.stratosphere.api.java.record.io.avro.AvroRecordInputFormatTest
  • eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample$ConcatenatingReducer
  • eu.stratosphere.api.java.record.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.CsvOutputFormatTest
  • eu.stratosphere.api.java.record.io.ExternalProcessFixedLengthInputFormatTest
  • eu.stratosphere.api.java.record.io.ExternalProcessInputFormatTest
  • eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.