Package org.apache.pig.data

Examples of org.apache.pig.data.InternalMap

668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
        Assert.assertEquals(2, t.size());

        Assert.assertEquals("1", t.get(0).toString());

        InternalMap map = new InternalMap();
        map.put("col1:cq1", new DataByteArray("value1"));
        map.put("col1:cq2", new DataByteArray("value2"));
        map.put("col1:cq3", new DataByteArray("value3"));
        map.put("col2:cq1", new DataByteArray("value1"));
        map.put("col3:cq1", new DataByteArray("value1"));
        map.put("col3:cq2", new DataByteArray("value2"));

        Assert.assertEquals(map, t.get(1));
    }
View Full Code Here
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
        Assert.assertEquals(2, t.size());

        Assert.assertEquals("1", t.get(0).toString());

        InternalMap map = new InternalMap();
        map.put("col1:cq1", new DataByteArray("value1"));
        map.put("col1:cq2", new DataByteArray("value2"));
        map.put("col1:cq3", new DataByteArray("value3"));
        map.put("col2:cq1", new DataByteArray("value1"));
        map.put("col3:cq1", new DataByteArray("value1"));
        map.put("col3:cq2", new DataByteArray("value2"));

        Assert.assertEquals(map, t.get(1));
    }
View Full Code Here
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
        // write it, both via DataReaderWriter and Tuple.readFields
        TupleFactory tf = TupleFactory.getInstance();

        Tuple t1 = tf.newTuple(1);

        InternalMap map = new InternalMap(2);
        map.put(new Integer(1), new String("world"));
        map.put(new Long(3L), new String("all"));
        t1.set(0, map);

        File file = File.createTempFile("Tuple", "put");
        FileOutputStream fos = new FileOutputStream(file);
        DataOutput out = new DataOutputStream(fos);
        t1.write(out);
        fos.close();

        FileInputStream fis = new FileInputStream(file);
        DataInput in = new DataInputStream(fis);

        Tuple after = tf.newTuple();
        after.readFields(in);

        Object o = after.get(0);
        assertTrue("isa InternalMap", o instanceof InternalMap);

        InternalMap m = (InternalMap)o;
        assertEquals("world", (String)m.get(new Integer(1)));
        assertEquals("all", (String)m.get(new Long(3L)));
        assertNull(m.get("fred"));

        file.delete();
    }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
    private float getProbVecSum(int numSamples, int numReduceres) throws Exception {
        DataBag samples = generateUniqueSamples(numSamples);
        Map<String, Object> res = getFindQuantilesResult(samples, numReduceres);

        InternalMap weightedPartsData = (InternalMap) res.get(FindQuantiles.WEIGHTED_PARTS);
        Iterator<Object> it = weightedPartsData.values().iterator();
        float[] probVec = getProbVec((Tuple)it.next());
        new DiscreteProbabilitySampleGenerator(probVec);
        float sum = 0.0f;
        for (float f : probVec) {
            sum += f;
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
                // the Quantiles file has a tuple as under:
                // (numQuantiles, bag of samples)
                // numQuantiles here is the reduce parallelism
                Map<String, Object> quantileMap = (Map<String, Object>) t.get(0);
                quantilesList = (DataBag) quantileMap.get(FindQuantiles.QUANTILES_LIST);
                InternalMap weightedPartsData = (InternalMap) quantileMap.get(FindQuantiles.WEIGHTED_PARTS);
                convertToArray(quantilesList);
                for(Entry<Object, Object> ent : weightedPartsData.entrySet()){
                    Tuple key = (Tuple)ent.getKey(); // sample item which repeats
                    float[] probVec = getProbVec((Tuple)ent.getValue());
                    weightedParts.put(getPigNullableWritable(key),
                            new DiscreteProbabilitySampleGenerator(probVec));
                }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
147
148
                // the Quantiles file has a tuple as under:
                // (numQuantiles, bag of samples)
                // numQuantiles here is the reduce parallelism
                Map<String, Object> quantileMap = (Map<String, Object>) t.get(0);
                quantilesList = (DataBag) quantileMap.get(FindQuantiles.QUANTILES_LIST);
                InternalMap weightedPartsData = (InternalMap) quantileMap.get(FindQuantiles.WEIGHTED_PARTS);
                convertToArray(quantilesList);
                for(Entry<Object, Object> ent : weightedPartsData.entrySet()){
                    Tuple key = (Tuple)ent.getKey(); // sample item which repeats
                    float[] probVec = getProbVec((Tuple)ent.getValue());
                    weightedParts.put(getPigNullableWritable(key),
                            new DiscreteProbabilitySampleGenerator(probVec));
                }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
147
148
                // the Quantiles file has a tuple as under:
                // (numQuantiles, bag of samples)
                // numQuantiles here is the reduce parallelism
                Map<String, Object> quantileMap = (Map<String, Object>) t.get(0);
                quantilesList = (DataBag) quantileMap.get(FindQuantiles.QUANTILES_LIST);
                InternalMap weightedPartsData = (InternalMap) quantileMap.get(FindQuantiles.WEIGHTED_PARTS);
                convertToArray(quantilesList);
                for(Entry<Object, Object> ent : weightedPartsData.entrySet()){
                    Tuple key = (Tuple)ent.getKey(); // sample item which repeats
                    float[] probVec = getProbVec((Tuple)ent.getValue());
                    weightedParts.put(getPigNullableWritable(key),
                            new DiscreteProbabilitySampleGenerator(probVec));
                }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
                // the Quantiles file has a tuple as under:
                // (numQuantiles, bag of samples)
                // numQuantiles here is the reduce parallelism
                Map<String, Object> quantileMap = (Map<String, Object>) t.get(0);
                quantilesList = (DataBag) quantileMap.get(FindQuantiles.QUANTILES_LIST);
                InternalMap weightedPartsData = (InternalMap) quantileMap.get(FindQuantiles.WEIGHTED_PARTS);
                convertToArray(quantilesList);
                for(Entry<Object, Object> ent : weightedPartsData.entrySet()){
                    Tuple key = (Tuple)ent.getKey(); // sample item which repeats
                    float[] probVec = getProbVec((Tuple)ent.getValue());
                    weightedParts.put(getPigNullableWritable(key),
                            new DiscreteProbabilitySampleGenerator(probVec));
                }
View Full Code Here
TOP

Related Classes of org.apache.pig.data.InternalMap

  • org.apache.pig.backend.hadoop.accumulo.TestAccumuloStorage
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.tez.runtime.WeightedRangePartitionerTez
  • org.apache.pig.impl.builtin.FindQuantiles
  • org.apache.pig.piggybank.storage.hiverc.HiveRCSchemaUtil
  • org.apache.pig.test.TestDataModel
  • org.apache.pig.test.TestFindQuantiles
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.