Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.DatumWrapper

142
143
144
145
146
147
148
149
150
151
152
153
154
  }

  private int compareInBinary1(SortComparator comp, ITuple tuple1, ITuple tuple2)
      throws IOException {
    DataOutputBuffer buffer1 = new DataOutputBuffer();
    ser.ser(new DatumWrapper(tuple1), buffer1);

    DataOutputBuffer buffer2 = new DataOutputBuffer();
    ser.ser(new DatumWrapper(tuple2), buffer2);

    return comp.compare(buffer1.getData(), 0, buffer1.getLength(), buffer2.getData(), 0,
        buffer2.getLength());
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    // Check that for the same prefix (number1, string1) we obtain the same partition
   
    int partitionId = -N_PARTITIONS;
    for(int i = 0; i < MAX_ITERATIONS_OVER_ONE_SCHEMA; i++) {
      tuple.set("string2", TestUtils.randomString(10));
      int thisPartitionId = partitioner.getPartition(new DatumWrapper(tuple), NullWritable.get(), N_PARTITIONS);
      Assert.assertTrue(thisPartitionId >= 0);
      Assert.assertTrue(thisPartitionId < N_PARTITIONS);
      if(partitionId == -N_PARTITIONS) {
        partitionId = thisPartitionId;
      } else {
        // Check that the returned partition is always the same even if "string2" field changes its value
        Assert.assertEquals(thisPartitionId, partitionId);
      }
    }
   
    // On the other hand, check that when we vary one of the group by fields, partition varies
   
    int partitionMatches[] = new int[N_PARTITIONS];
    for(int i = 0; i < MAX_ITERATIONS_OVER_ONE_SCHEMA; i++) {
      tuple.set("string1", TestUtils.randomString(10));
      int thisPartitionId = partitioner.getPartition(new DatumWrapper(tuple), NullWritable.get(), N_PARTITIONS);
      Assert.assertTrue(thisPartitionId >= 0);
      Assert.assertTrue(thisPartitionId < N_PARTITIONS);
      partitionMatches[thisPartitionId]++;;
    }
   
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
   
    int partitionMatches[] = new int[N_PARTITIONS];
   
    for(int i = 0; i < MAX_ITERATIONS_OVER_ONE_SCHEMA; i++) {
      tuple.set("foo", (int)(Math.random() * Integer.MAX_VALUE));
      int thisPartitionId = partitioner.getPartition(new DatumWrapper(tuple), NullWritable.get(), N_PARTITIONS);
      Assert.assertTrue(thisPartitionId >= 0);
      Assert.assertTrue(thisPartitionId < N_PARTITIONS);
      partitionMatches[thisPartitionId]++;;
    }
   
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.DatumWrapper

  • com.datasalt.pangool.tuplemr.mapred.TestComparators
  • com.datasalt.pangool.tuplemr.mapred.TestTupleHashPartitioner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.