Package org.apache.pig.ResourceSchema

Examples of org.apache.pig.ResourceSchema.ResourceFieldSchema

426
427
428
429
430
431
432
433
434
435
436
437
    @SuppressWarnings("unchecked")
    public Map<String, Object> bytesToMap(byte[] b) throws IOException {
        if(b == null)
            return null;
        Map<String, Object> map;
        ResourceFieldSchema fs = new ResourceFieldSchema();
        fs.setType(DataType.MAP);
        try {
            ByteArrayInputStream bis = new ByteArrayInputStream(b);
            PushbackInputStream in = new PushbackInputStream(bis);
            map = consumeMap(in, fs);
        }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
        if (!descending) {
          ResourceSchema.ResourceFieldSchema[] fields = schema.getFields();
          int[] index = schema.getSortKeys();
       
          for( int i = 0; i< index.length; i++ ) {
            ResourceFieldSchema field = fields[index[i]];
            String name = field.getName();
            if( name == null )
                throw new IOException("Zebra does not support column positional reference yet");
            if( !org.apache.pig.data.DataType.isAtomic( field.getType() ) )
                throw new IOException( "Field [" + name + "] is not of simple type as required for a sort column now." );
            if( i > 0 )
                sortColumnNames.append( "," );
            sortColumnNames.append( name );
          }
View Full Code Here
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
        // will contain all fields for this schema
        List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

        for (ColumnDef cdef : cfDef.column_metadata)
        {
            ResourceFieldSchema valSchema = new ResourceFieldSchema();
            AbstractType validator = validators.get(cdef.name);
            if (validator == null)
                validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
            valSchema.setName(new String(cdef.getName()));
            valSchema.setType(getPigType(validator));
            allSchemaFields.add(valSchema);
        }

        // top level schema contains everything
        schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
        if(r==null) return new DataByteArray("RANDOM".getBytes());
        return new DataByteArray(genRandString(r).getBytes());
    }
   
    public static ResourceFieldSchema getSmallTupleFieldSchema() throws IOException{
        ResourceFieldSchema stringfs = new ResourceFieldSchema();
        stringfs.setType(DataType.CHARARRAY);
        ResourceFieldSchema intfs = new ResourceFieldSchema();
        intfs.setType(DataType.INTEGER);
       
        ResourceSchema tupleSchema = new ResourceSchema();
        tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, intfs});
        ResourceFieldSchema tuplefs = new ResourceFieldSchema();
        tuplefs.setSchema(tupleSchema);
        tuplefs.setType(DataType.TUPLE);
       
        return tuplefs;
    }
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
       
        return db;
    }
   
    public static ResourceFieldSchema getSmallTupDataBagFieldSchema() throws IOException {
        ResourceFieldSchema tuplefs = getSmallTupleFieldSchema();
       
        ResourceSchema bagSchema = new ResourceSchema();
        bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
        ResourceFieldSchema bagfs = new ResourceFieldSchema();
        bagfs.setSchema(bagSchema);
        bagfs.setType(DataType.BAG);
       
        return bagfs;
    }
View Full Code Here
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
        t.append(genRandSmallTuple(r, 100));
        return t;
    }
   
    public static ResourceFieldSchema getSmallBagTextTupleFieldSchema() throws IOException{
        ResourceFieldSchema stringfs = new ResourceFieldSchema();
        stringfs.setType(DataType.CHARARRAY);
       
        ResourceFieldSchema intfs = new ResourceFieldSchema();
        intfs.setType(DataType.INTEGER);
     
        ResourceFieldSchema bagfs = getSmallTupDataBagFieldSchema();
       
        ResourceFieldSchema floatfs = new ResourceFieldSchema();
        floatfs.setType(DataType.FLOAT);
       
        ResourceFieldSchema doublefs = new ResourceFieldSchema();
        doublefs.setType(DataType.DOUBLE);
       
        ResourceFieldSchema longfs = new ResourceFieldSchema();
        longfs.setType(DataType.LONG);
       
        ResourceFieldSchema mapfs = new ResourceFieldSchema();
        mapfs.setType(DataType.MAP);
       
        ResourceFieldSchema tuplefs = getSmallTupleFieldSchema();
       
        ResourceSchema outSchema = new ResourceSchema();
        outSchema.setFields(new ResourceFieldSchema[]{bagfs, stringfs, stringfs, doublefs, floatfs,
                intfs, longfs, mapfs, tuplefs});
        ResourceFieldSchema outfs = new ResourceFieldSchema();
        outfs.setSchema(outSchema);
        outfs.setType(DataType.TUPLE);
       
        return outfs;
    }
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
        }
        return db;
    }

    public static ResourceFieldSchema getFullTupTextDataBagFieldSchema() throws IOException{
        ResourceFieldSchema tuplefs = getSmallBagTextTupleFieldSchema();
       
        ResourceSchema outBagSchema = new ResourceSchema();
        outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
        ResourceFieldSchema outBagfs = new ResourceFieldSchema();
        outBagfs.setSchema(outBagSchema);
        outBagfs.setType(DataType.BAG);
       
        return outBagfs;
    }
View Full Code Here
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
        }
        return db;
    }
   
    public static ResourceFieldSchema getFloatDataBagFieldSchema(int column) throws IOException {
        ResourceFieldSchema intfs = new ResourceFieldSchema();
        intfs.setType(DataType.INTEGER);
       
        ResourceSchema tupleSchema = new ResourceSchema();
        ResourceFieldSchema[] fss = new ResourceFieldSchema[column];
        for (int i=0;i<column;i++) {
            fss[i] = intfs;
        }
        tupleSchema.setFields(fss);
        ResourceFieldSchema tuplefs = new ResourceFieldSchema();
        tuplefs.setSchema(tupleSchema);
        tuplefs.setType(DataType.TUPLE);
       
        ResourceSchema bagSchema = new ResourceSchema();
        bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
        ResourceFieldSchema bagfs = new ResourceFieldSchema();
        bagfs.setSchema(bagSchema);
        bagfs.setType(DataType.BAG);
       
        return bagfs;
    }
View Full Code Here
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
            t.append("false");
        return t;
    }
   
    public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() throws IOException {
        ResourceFieldSchema stringfs = new ResourceFieldSchema();
        stringfs.setType(DataType.CHARARRAY);
        ResourceFieldSchema intfs = new ResourceFieldSchema();
        intfs.setType(DataType.INTEGER);
        ResourceFieldSchema longfs = new ResourceFieldSchema();
        longfs.setType(DataType.LONG);
        ResourceFieldSchema floatfs = new ResourceFieldSchema();
        floatfs.setType(DataType.FLOAT);
        ResourceFieldSchema doublefs = new ResourceFieldSchema();
        doublefs.setType(DataType.DOUBLE);
        ResourceFieldSchema booleanfs = new ResourceFieldSchema();
        booleanfs.setType(DataType.BOOLEAN);
       
        ResourceSchema tupleSchema = new ResourceSchema();
        tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, longfs, intfs, doublefs, floatfs, stringfs, intfs, doublefs, floatfs, booleanfs});
        ResourceFieldSchema tuplefs = new ResourceFieldSchema();
        tuplefs.setSchema(tupleSchema);
        tuplefs.setType(DataType.TUPLE);
       
        return tuplefs;
    }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
    public  void testBytesToTuple() throws IOException
    {
        for (int i = 0; i < MAX; i++) {
            Tuple t = GenRandomData.genRandSmallBagTextTuple(r, 1, 100);
           
            ResourceFieldSchema fs = GenRandomData.getSmallBagTextTupleFieldSchema();
           
            Tuple convertedTuple = ps.getLoadCaster().bytesToTuple(t.toString().getBytes(), fs);
            assertTrue(TestHelper.tupleEquals(t, convertedTuple));
        }
       
View Full Code Here
TOP

Related Classes of org.apache.pig.ResourceSchema.ResourceFieldSchema

  • com.netflix.aegisthus.pig.AegisthusLoader
  • com.twitter.elephantbird.pig.load.SequenceFileLoader
  • com.twitter.elephantbird.pig.mahout.VectorWritableConverter
  • com.twitter.elephantbird.pig.util.BytesWritableConverter
  • com.twitter.elephantbird.pig.util.GenericWritableConverter
  • com.twitter.elephantbird.pig.util.IntWritableConverter
  • com.twitter.elephantbird.pig.util.LongWritableConverter
  • com.twitter.elephantbird.pig.util.NullWritableConverter
  • com.twitter.elephantbird.pig.util.ProtobufWritableConverter
  • com.twitter.elephantbird.pig.util.ResourceSchemaUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.