Examples of org.apache.pig.ResourceSchema

org.apache.pig.ResourceSchema
A represenation of a schema used to communicate with load and store functions. This is separate from {@link Schema}, which is an internal Pig representation of a schema. @since Pig 0.7


        if(includeDayDateColumn){
            fieldschema.add(new FieldSchema(DAY_DATE_COLUMN, DataType.CHARARRAY));
        }


        pigSchema = new ResourceSchema(new Schema(fieldschema));


        props =  new Properties();


        //   setting table schema properties for ColumnarSerDe
        //   these properties are never changed by the columns to read filter, because the columnar serde needs to now the

View Full Code Here

        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()
                                .getFileName(), new Job(conf));
            }
        }
    }

View Full Code Here

        ResourceFieldSchema stringfs = new ResourceFieldSchema();
        stringfs.setType(DataType.CHARARRAY);
        ResourceFieldSchema intfs = new ResourceFieldSchema();
        intfs.setType(DataType.INTEGER);
        
        ResourceSchema tupleSchema = new ResourceSchema();
        tupleSchema.setFields(new ResourceFieldSchema[]{intfs, stringfs});
        ResourceFieldSchema tuplefs = new ResourceFieldSchema();
        tuplefs.setSchema(tupleSchema);
        tuplefs.setType(DataType.TUPLE);
        
        return tuplefs;

View Full Code Here

    }
    
    public ResourceFieldSchema getBagFieldSchema() throws IOException{
        ResourceFieldSchema tuplefs = getTupleFieldSchema();
        
        ResourceSchema outBagSchema = new ResourceSchema();
        outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
        ResourceFieldSchema outBagfs = new ResourceFieldSchema();
        outBagfs.setSchema(outBagSchema);
        outBagfs.setType(DataType.BAG);
        
        return outBagfs;

View Full Code Here

        }


        @Override
        public ResourceSchema getSchema(String location, Job job)
                throws IOException {
            return new ResourceSchema(schema);
        }

View Full Code Here

            ResourceFieldSchema stringfs = new ResourceFieldSchema();
            stringfs.setType(DataType.CHARARRAY);
            ResourceFieldSchema intfs = new ResourceFieldSchema();
            intfs.setType(DataType.INTEGER);
            
            ResourceSchema tupleSchema = new ResourceSchema();
            tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, intfs});
            ResourceFieldSchema tuplefs = new ResourceFieldSchema();
            tuplefs.setSchema(tupleSchema);
            tuplefs.setType(DataType.TUPLE);
            
            ResourceSchema bagSchema = new ResourceSchema();
            bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
            ResourceFieldSchema bagfs = new ResourceFieldSchema();
            bagfs.setSchema(bagSchema);
            bagfs.setType(DataType.BAG);
            
            t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes(), bagfs) : null);

View Full Code Here

                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName, 
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Util.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            Assert.assertTrue("Checking binstorage getSchema output", Schema.equals(

View Full Code Here

                // the create() below will fail is this method gets called
                // more than once
                if(!new File(checkFileName).createNewFile()) {
                    throw new RuntimeException(checkFileName + " already exists!");
                }
                return new ResourceSchema(
                        Utils.getSchemaFromString("a:chararray,b:int"));
            } catch (ParseException e) {
                throw new IOException(e);
            }
        }

View Full Code Here

    for (HCatFieldSchema hfs : hcatSchema.getFields()) {
      ResourceFieldSchema rfSchema;
      rfSchema = getResourceSchemaFromFieldSchema(hfs);
      rfSchemaList.add(rfSchema);
    }
    ResourceSchema rSchema = new ResourceSchema();
    rSchema.setFields(rfSchemaList.toArray(new ResourceFieldSchema[0]));
    return rSchema;


  }

View Full Code Here

      .setType(DataType.TUPLE);
    HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0);
    if (arrayElementFieldSchema.getType() == Type.STRUCT) {
      bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema));
    } else if (arrayElementFieldSchema.getType() == Type.ARRAY) {
      ResourceSchema s = new ResourceSchema();
      List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema));
      s.setFields(lrfs.toArray(new ResourceFieldSchema[0]));
      bagSubFieldSchemas[0].setSchema(s);
    } else {
      ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1];
      innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName)
        .setDescription("The inner field in the tuple in the bag")
        .setType(getPigType(arrayElementFieldSchema))
        .setSchema(null); // the element type is not a tuple - so no subschema
      bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
    }
    ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas);
    return s;


  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.ResourceSchema

com.acme.io.JsonLoader

com.acme.io.JsonStorage

com.linkedin.pig.FastAvroStorageCommon

com.mongodb.hadoop.pig.BSONLoader

com.mongodb.hadoop.pig.BSONStorage

com.mongodb.hadoop.pig.BSONStorageTest

com.mongodb.hadoop.pig.JSONPigReplace

com.mongodb.hadoop.pig.MongoInsertStorage

com.mongodb.hadoop.pig.MongoLoader

com.mongodb.hadoop.pig.MongoStorage

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.