Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.NullableTuple

381
382
383
384
385
386
387
388
389
390
    @Test
    public void testRandomTuples() throws IOException {
        Random rand = new Random(SEED);
        for (int i = 0; i < TUPLE_NUMBER; i++) {
            NullableTuple t = new NullableTuple(getRandomTuple(rand));
            int res = compareHelper(prototype, t, comparator);
            assertEquals(Math.signum(prototype.compareTo(t)), Math.signum(res), 0);
        }
    }
View Full Code Here
391
392
393
394
395
396
397
398
399
400
401
    @Test
    public void testSortOrder() throws IOException {
        // prototype < t but we use inverse sort order
        list.set(2, (Double) list.get(2) + 0.1);
        NullableTuple t = new NullableTuple(tf.newTuple(list));
        JobConf jobConf = new JobConf();
        jobConf.set("pig.sortOrder", ObjectSerializer.serialize(new boolean[] {false}));
        comparator.setConf(jobConf);
        int res = compareHelper(prototype, t, comparator);
        assertEquals(-1 * Math.signum(prototype.compareTo(t)), Math.signum(res), 0);
View Full Code Here
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
        Random rand = new Random(SEED);
        TestPigTupleRawComparator test = new TestPigTupleRawComparator();
        test.setUp();
        byte[][] toCompare1 = new byte[TUPLE_NUMBER][];
        byte[][] toCompare2 = new byte[TUPLE_NUMBER][];
        NullableTuple t;
        for (int i = 0; i < TUPLE_NUMBER; i++) {
            t = new NullableTuple(test.getRandomTuple(rand));
            t.write(test.dos1);
            toCompare1[i] = test.baos1.toByteArray();
        }
        for (int i = 0; i < TUPLE_NUMBER; i++) {
            t = new NullableTuple(test.getRandomTuple(rand));
            t.write(test.dos2);
            toCompare2[i] = test.baos2.toByteArray();
        }

        before = System.currentTimeMillis();
        for (int loop = 0; loop < TIMES; loop++) {
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
        try {
            // user comparators work with tuples - so if user comparator
            // is being used OR if there are more than 1 sort cols, use
            // NullableTuple
            if ("true".equals(job.get("pig.usercomparator")) || t.size() > 1) {
                return new NullableTuple(t);
            } else {
                Object o = t.get(0);
                String kts = job.get("pig.reduce.key.type");
                if (kts == null) {
                    throw new RuntimeException("Didn't get reduce key type "
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
        return rc;
    }

    public int compare(Object o1, Object o2) {
        NullableTuple nt1 = (NullableTuple) o1;
        NullableTuple nt2 = (NullableTuple) o2;
        int rc = 0;

        // If either are null, handle differently.
        if (!nt1.isNull() && !nt2.isNull()) {
            rc = compareTuple((Tuple) nt1.getValueAsPigType(), (Tuple) nt2.getValueAsPigType());
        } else {
            // For sorting purposes two nulls are equal.
            if (nt1.isNull() && nt2.isNull())
                rc = 0;
            else if (nt1.isNull())
                rc = -1;
            else
                rc = 1;
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
                throws InterruptedException, IOException {
           
            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(1), keyType);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
           
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);          
             
            oc.write(key, val);
        }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
           

            Byte index = (Byte)tuple.get(0);
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(keyTuple, DataType.TUPLE);
            NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
           
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            key.setIndex(index);
            val.setIndex(index);

            oc.write(key, val);
        }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            PigNullableWritable key =
                HDataType.getWritableComparableTypes(tuple.get(tupleKeyIdx), DataType.TUPLE);

            NullablePartitionWritable wrappedKey = new NullablePartitionWritable(key);

            NullableTuple val = new NullableTuple((Tuple)tuple.get(tupleValIdx));
           
            // Both the key and the value need the index.  The key needs it so
            // that it can be sorted on the index in addition to the key
            // value.  The value needs it so that POPackage can properly
            // assign the tuple to its slot in the projection.
            wrappedKey.setIndex(index);
           
            // set the partition
            wrappedKey.setPartition(partitionIndex);
            val.setIndex(index);

            oc.write(wrappedKey, val);
        }
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
                        if(redRes.returnStatus==POStatus.STATUS_OK){
                            Tuple tuple = (Tuple)redRes.result;
                            Byte index = (Byte)tuple.get(0);
                            PigNullableWritable outKey =
                                HDataType.getWritableComparableTypes(tuple.get(1), this.keyType);
                            NullableTuple val =
                                new NullableTuple((Tuple)tuple.get(2));
                            // Both the key and the value need the index.  The key needs it so
                            // that it can be sorted on the index in addition to the key
                            // value.  The value needs it so that POPackage can properly
                            // assign the tuple to its slot in the projection.
                            outKey.setIndex(index);
                            val.setIndex(index);

                            oc.write(outKey, val);

                            continue;
                        }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
        DataBag db2 = GenRandomData.genRandSmallTupDataBag(r, 10, 100);
        List<NullableTuple> db = new ArrayList<NullableTuple>(200);
        Iterator db1Iter = db1.iterator();
        if(!inner[0]){
            while (db1Iter.hasNext()) {
                NullableTuple it = new NullableTuple((Tuple)db1Iter.next());
                it.setIndex((byte)0);
                db.add(it);
            }
        }
        Iterator db2Iter = db2.iterator();
        while (db2Iter.hasNext()) {
            NullableTuple it = new NullableTuple((Tuple) db2Iter.next());
            it.setIndex((byte)1);
            db.add(it);
        }
        //ITIterator iti = new TestPackage.ITIterator(db.iterator());
        POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
        pop.setNumInps(2);
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.NullableTuple

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.DistinctCombiner$Combine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigCombiner$Combine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Map
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$MapWithComparator
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$MapWithPartitionIndex
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Map
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$MapWithComparator
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$MapWithPartitionIndex
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSecondaryKeyComparator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.