Package com.ebay.erl.mobius.core.datajoin

Examples of com.ebay.erl.mobius.core.datajoin.DataJoinKey

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
            // joinmap function might generate more than one output key
            // per <code>key</code> input.
            for( Tuple t:collector.getOutKey() )
            {
              Tuple mt = Tuple.merge(tuple, t);
              DataJoinKey nkey = this.getKey(mt, sorters, datasetID, mapper, job);
              samples.add(nkey);
            }
          }
          else
          {
            // When exceeding the maximum number of samples, replace
            // a random element with this one, then adjust the
            // frequency to reflect the possibility of existing
            // elements being pushed out
           
            mapper.joinmap(key, value, collector, Reporter.NULL);
            for( Tuple t:collector.getOutKey() )
            {
              int ind = r.nextInt(numSamples);
              if (ind != numSamples)
              {
                Tuple mt = Tuple.merge(tuple, t);
                DataJoinKey nkey = this.getKey(mt, sorters, datasetID, mapper, job);
                samples.set(ind, nkey);
              }
            }
           
            freq *= (numSamples - collector.getOutKey().size()) / (double) numSamples;
View Full Code Here
219
220
221
222
223
224
225
226
227
      String name    = aSorter.getColumn();
      Object value  = tuple.get(name);
      columnsUsedToSort.insert(name, value);
    }
   
    DataJoinKey nkey = new DataJoinKey(datasetID, columnsUsedToSort, mapper.extractSortValueKeyword(tuple), mapper.getSortValueComparator());
    nkey.setConf(conf);
    return nkey;
  }
View Full Code Here
190
191
192
193
194
195
196
197
198
199
            throw new IllegalArgumentException(p.toString()+" is a group function that generates " +
                "more than one rows ("+aggregatedResult.size()+") per key, so it is not combinable.");
        }
      }
     
      DataJoinKey outKey    = new DataJoinKey(datasetID, key);
      DataJoinValue outValue  = new DataJoinValue(datasetID, combinedValue);
      output.collect(outKey, outValue);
    }
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
*/
public class DataJoinKeyTest {

  @Test
  public void testConstructor() {
    DataJoinKey djKey = new DataJoinKey(Byte.valueOf("1"), new Text("test value"));
   
    Assert.assertEquals("1", djKey.getDatasetID());
    Assert.assertEquals(new Text("test value"), djKey.getKey());
    //Assert.assertEquals(null, djKey.getSortKeyword());
    //Assert.assertEquals(Class.class, djKey.getSortComparator());
   
    djKey = new DataJoinKey(Byte.valueOf("1"), new Text("test value"), new Text("2"), DataJoinKey.Comparator.class);
   
    Assert.assertEquals("1", djKey.getDatasetID());
    Assert.assertEquals(new Text("test value"), djKey.getKey());
    //Assert.assertEquals(new Text("2"), djKey.getSortKeyword());
    //Assert.assertEquals(DataJoinKey.Comparator.class, djKey.getSortComparator());
   
    djKey = new DataJoinKey(Byte.valueOf("1"), new Text("test value"), new Text("2"), null);
   
    Assert.assertEquals(Byte.valueOf("1"), djKey.getDatasetID());
    Assert.assertEquals(new Text("test value"), djKey.getKey());
    //Assert.assertEquals(new Text("2"), djKey.getSortKeyword());
    //Assert.assertEquals(Class.class, djKey.getSortComparator());
  }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
  }
 
 
  @Test
  public void testCompare() {
    DataJoinKey djKey1 = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey1_1 = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey2 = new DataJoinKey(Byte.valueOf("1"), new Text("2"));
   
    Assert.assertEquals(-1, djKey1.compareTo(djKey2));
    Assert.assertEquals(1, djKey2.compareTo(djKey1));
    Assert.assertEquals(0, djKey1.compareTo(djKey1_1));
   
    Assert.assertEquals(-1, djKey1.compare(djKey1, djKey2));
    Assert.assertEquals(1, djKey1.compare(djKey2, djKey1));
    Assert.assertEquals(0, djKey1.compare(djKey1, djKey1_1));
   
    djKey1    = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("1"), null);
    djKey1_1   = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("2"), null);
    djKey2     = new DataJoinKey(Byte.valueOf("1"), new Text("2"), new Text("1"), null);
   
    Assert.assertEquals(-1, djKey1.compareTo(djKey2));
    Assert.assertEquals(1, djKey2.compareTo(djKey1));
    Assert.assertEquals(0, djKey1.compareTo(djKey1_1));
    Assert.assertEquals(0, djKey1_1.compareTo(djKey1));
   
    Assert.assertEquals(-1, djKey1.compare(djKey1, djKey2));
    Assert.assertEquals(1, djKey1.compare(djKey2, djKey1));
    Assert.assertEquals(0, djKey1.compare(djKey1, djKey1_1));
    Assert.assertEquals(0, djKey1.compare(djKey1_1, djKey1));
   
    djKey1     = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("1"), NegativeComparator.class);
    djKey1_1   = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("2"), NegativeComparator.class);
    djKey2     = new DataJoinKey(Byte.valueOf("1"), new Text("2"), new Text("1"), NegativeComparator.class);
   
    Assert.assertEquals(-1, djKey1.compareTo(djKey2));
    Assert.assertEquals(1, djKey2.compareTo(djKey1));
    Assert.assertEquals(0, djKey1.compareTo(djKey1_1));
    Assert.assertEquals(0, djKey1_1.compareTo(djKey1));
   
    Assert.assertEquals(-1, djKey1.compare(djKey1, djKey2));
    Assert.assertEquals(1, djKey1.compare(djKey2, djKey1));
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
    Assert.assertEquals(0, djKey1.compare(djKey1_1, djKey1));
  }
 
  @Test
  public void testHash() {
    DataJoinKey djKey1     = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey1_1   = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey2     = new DataJoinKey(Byte.valueOf("1"), new Text("2"));
   
    Assert.assertEquals(djKey1.hashCode(), djKey1_1.hashCode());
    Assert.assertNotSame(djKey1.hashCode(), djKey2.hashCode());
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    t1.put("k1", "v1");
   
    Tuple t2 = new Tuple();
    t2.put("k2", "v2");
   
    DataJoinKey k1 = new DataJoinKey(Byte.valueOf("1"), t1);
    DataJoinKey k2 = new DataJoinKey(Byte.valueOf("1"), t2);
   
    ByteArrayOutputStream b1 = new ByteArrayOutputStream();
    ByteArrayOutputStream b2 = new ByteArrayOutputStream();
   
    DataOutputStream out1 = new DataOutputStream(b1);
    DataOutputStream out2 = new DataOutputStream(b2);
   
    k1.write(out1);
    k2.write(out2);
   
    out1.flush();
    out2.flush();
   
    byte[] ba1 = b1.toByteArray();
    byte[] ba2 = b2.toByteArray();
   
    DataInputStream in1 = new DataInputStream(new ByteArrayInputStream(ba1));
    DataInputStream in2 = new DataInputStream(new ByteArrayInputStream(ba2));
   
    DataJoinKey k_1 = new DataJoinKey();
    k_1.readFields(in1);
   
    DataJoinKey k_2 = new DataJoinKey();
    k_2.readFields(in2);
   
    Assert.assertEquals(-1, k_1.compareTo(k_2));
    Assert.assertEquals(-1, k_1.compare(ba1, 0, ba1.length, ba2, 0, ba2.length));
   
  }
View Full Code Here
24
25
26
27
28
29
30
31
32
33
34
35
36
  @Test
  public void testPartition() {
    DataJoinKeyPartitioner partitioner = new DataJoinKeyPartitioner();
   
    DataJoinKey djKey1 = new DataJoinKey(Byte.valueOf("1"), new Text("test value"));
    DataJoinKey djKey2 = new DataJoinKey(Byte.valueOf("1"), new Text("test value"));
    DataJoinKey djKey3 = new DataJoinKey(Byte.valueOf("1"), new Text("other test value"));
   
    System.out.println(partitioner.getPartition(djKey1, null, 999));
    System.out.println(partitioner.getPartition(djKey2, null, 999));
    System.out.println(partitioner.getPartition(djKey3, null, 999));
   
View Full Code Here
TOP

Related Classes of com.ebay.erl.mobius.core.datajoin.DataJoinKey

  • com.ebay.erl.mobius.core.mapred.DefaultMobiusCombiner
  • com.ebay.erl.mobius.core.mapred.MobiusInputSampler
  • com.ebay.erl.mobius.core.model.Tuple
  • com.ebay.erl.mobius.datajoin.DataJoinKeyPartitionerTest
  • com.ebay.erl.mobius.datajoin.DataJoinKeyTest
  • org.apache.hadoop.io.DataInputBuffer
  • org.apache.hadoop.io.WritableComparable
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.