Package org.apache.pig

Examples of org.apache.pig.ResourceSchema$ResourceFieldSchema

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    private PhoenixPigSchemaUtil() {
    }
   
    public static ResourceSchema getResourceSchema(final PhoenixPigConfiguration phoenixConfiguration) throws IOException {
       
        final ResourceSchema schema = new ResourceSchema();
        try {
          List<ColumnInfo> columns = null;
          if(SchemaType.QUERY.equals(phoenixConfiguration.getSchemaType())) {
            final String sqlQuery = phoenixConfiguration.getSelectStatement();
            Preconditions.checkNotNull(sqlQuery, "No Sql Query exists within the configuration");
            final SqlQueryToColumnInfoFunction function = new SqlQueryToColumnInfoFunction(phoenixConfiguration);
            columns = function.apply(sqlQuery);
          } else {
            columns = phoenixConfiguration.getSelectColumnMetadataList();
          }
            ResourceFieldSchema fields[] = new ResourceFieldSchema[columns.size()];
            int i = 0;
            for(ColumnInfo cinfo : columns) {
                int sqlType = cinfo.getSqlType();
                PDataType phoenixDataType = PDataType.fromTypeId(sqlType);
                byte pigType = TypeUtil.getPigDataTypeForPhoenixType(phoenixDataType);
                ResourceFieldSchema field = new ResourceFieldSchema();
                field.setType(pigType).setName(cinfo.getDisplayName());
                fields[i++] = field;
            }
            schema.setFields(fields);   
        } catch(SQLException sqle) {
            LOG.error(String.format("Error: SQLException [%s] ",sqle.getMessage()));
            throw new IOException(sqle);
        }
       
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    public void testSchema() throws SQLException, IOException {
       
        final PhoenixPigConfiguration configuration = mock(PhoenixPigConfiguration.class);
        final List<ColumnInfo> columnInfos = ImmutableList.of(ID_COLUMN,NAME_COLUMN);
        when(configuration.getSelectColumnMetadataList()).thenReturn(columnInfos);
        final ResourceSchema actual = PhoenixPigSchemaUtil.getResourceSchema(configuration);
       
        // expected schema.
        final ResourceFieldSchema[] fields = new ResourceFieldSchema[2];
        fields[0] = new ResourceFieldSchema().setName("ID")
                                                .setType(DataType.LONG);

        fields[1] = new ResourceFieldSchema().setName("NAME")
                                                .setType(DataType.CHARARRAY);
        final ResourceSchema expected = new ResourceSchema().setFields(fields);
       
        assertEquals(expected.toString(), actual.toString());
       
    }
View Full Code Here
263
264
265
266
267
268
269
270
271
272
273
        }
        if(inputAvroSchema != null) {
            AvroStorageLog.details( "avro input schema:"  + inputAvroSchema);

            /* convert to pig schema */
            ResourceSchema pigSchema = AvroSchema2Pig.convert(inputAvroSchema);
            AvroStorageLog.details("pig input schema:" + pigSchema);
            return pigSchema;
        } else
            return null;
    }
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
318
        }
    }

    /** wrap a pig schema as tuple */
    public static ResourceFieldSchema wrapAsTuple(ResourceFieldSchema subFieldSchema) throws IOException {
        ResourceSchema listSchema = new ResourceSchema();
        listSchema.setFields(new ResourceFieldSchema[] { subFieldSchema });

        ResourceFieldSchema tupleWrapper = new ResourceFieldSchema();
        tupleWrapper.setType(DataType.TUPLE);
        tupleWrapper.setName(PIG_TUPLE_WRAPPER);
        tupleWrapper.setSchema(listSchema);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
        ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1];
        innerTupleFieldSchemas[0] =
            new ResourceFieldSchema().setName("innerfield").setType(DataType.CHARARRAY);

        bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
        ResourceSchema expected = new ResourceSchema().setFields(bagSubFieldSchemas);

        // Get the actual converted schema.
        HCatSchema hCatSchema = new HCatSchema(Lists.newArrayList(
            new HCatFieldSchema("innerLlama", HCatFieldSchema.Type.STRING, null)));
        HCatFieldSchema hCatFieldSchema =
            new HCatFieldSchema("llama", HCatFieldSchema.Type.ARRAY, hCatSchema, null);
        ResourceSchema actual = PigHCatUtil.getBagSubSchema(hCatFieldSchema);

        Assert.assertEquals(expected.toString(), actual.toString());
    }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
        ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1];
        innerTupleFieldSchemas[0] =
            new ResourceFieldSchema().setName("llama_tuple").setType(DataType.CHARARRAY);

        bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
        ResourceSchema expected = new ResourceSchema().setFields(bagSubFieldSchemas);

        // Get the actual converted schema.
        HCatSchema actualHCatSchema = new HCatSchema(Lists.newArrayList(
            new HCatFieldSchema("innerLlama", HCatFieldSchema.Type.STRING, null)));
        HCatFieldSchema actualHCatFieldSchema =
            new HCatFieldSchema("llama", HCatFieldSchema.Type.ARRAY, actualHCatSchema, null);
        ResourceSchema actual = PigHCatUtil.getBagSubSchema(actualHCatFieldSchema);

        Assert.assertEquals(expected.toString(), actual.toString());
    }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
        for (HCatFieldSchema hfs : hcatSchema.getFields()) {
            ResourceFieldSchema rfSchema;
            rfSchema = getResourceSchemaFromFieldSchema(hfs);
            rfSchemaList.add(rfSchema);
        }
        ResourceSchema rSchema = new ResourceSchema();
        rSchema.setFields(rfSchemaList.toArray(new ResourceFieldSchema[0]));
        return rSchema;

    }
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
            .setType(DataType.TUPLE);
        HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0);
        if (arrayElementFieldSchema.getType() == Type.STRUCT) {
            bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema));
        } else if (arrayElementFieldSchema.getType() == Type.ARRAY) {
            ResourceSchema s = new ResourceSchema();
            List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema));
            s.setFields(lrfs.toArray(new ResourceFieldSchema[0]));
            bagSubFieldSchemas[0].setSchema(s);
        } else {
            ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1];
            innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName)
                .setDescription("The inner field in the tuple in the bag")
                .setType(getPigType(arrayElementFieldSchema))
                .setSchema(null); // the element type is not a tuple - so no subschema
            bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
        }
        ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas);
        return s;

    }
View Full Code Here
277
278
279
280
281
282
283
284
285
286
287
288
289
    }

    private static ResourceSchema getTupleSubSchema(HCatFieldSchema hfs) throws IOException {
        // for each struct subfield, create equivalent ResourceFieldSchema
        ResourceSchema s = new ResourceSchema();
        List<ResourceFieldSchema> lrfs = new ArrayList<ResourceFieldSchema>();
        for (HCatFieldSchema subField : hfs.getStructSubSchema().getFields()) {
            lrfs.add(getResourceSchemaFromFieldSchema(subField));
        }
        s.setFields(lrfs.toArray(new ResourceFieldSchema[0]));
        return s;
    }
View Full Code Here
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
    {
        setLocation(location, job);
        CfInfo cfInfo = getCfInfo(loadSignature);
        CfDef cfDef = cfInfo.cfDef;
        // top-level schema, no type
        ResourceSchema schema = new ResourceSchema();

        // get default marshallers and validators
        Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
        Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

        // will contain all fields for this schema
        List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

        for (ColumnDef cdef : cfDef.column_metadata)
        {
            ResourceFieldSchema valSchema = new ResourceFieldSchema();
            AbstractType validator = validators.get(cdef.name);
            if (validator == null)
                validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
            valSchema.setName(new String(cdef.getName()));
            valSchema.setType(getPigType(validator));
            allSchemaFields.add(valSchema);
        }

        // top level schema contains everything
        schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
        return schema;
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.ResourceSchema$ResourceFieldSchema

  • com.acme.io.JsonLoader
  • com.acme.io.JsonStorage
  • com.linkedin.pig.FastAvroStorageCommon
  • com.mongodb.hadoop.pig.BSONLoader
  • com.mongodb.hadoop.pig.BSONStorage
  • com.mongodb.hadoop.pig.BSONStorageTest
  • com.mongodb.hadoop.pig.JSONPigReplace
  • com.mongodb.hadoop.pig.MongoInsertStorage
  • com.mongodb.hadoop.pig.MongoLoader
  • com.mongodb.hadoop.pig.MongoStorage
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.