Examples of org.apache.pig.ResourceSchema

org.apache.pig.ResourceSchema
A represenation of a schema used to communicate with load and store functions. This is separate from {@link Schema}, which is an internal Pig representation of a schema. @since Pig 0.7

        Schema level0 = TypeCheckingTestUtil.genFlatSchema(
                aliases,types);
        Schema.FieldSchema fld0 = 
            new Schema.FieldSchema("f0", level0, DataType.BAG);
        Schema level1 = new Schema(fld0);
        Schema genSchema = Schema.getPigSchema(new ResourceSchema(level1));
        assertTrue(CheckTwoLevelAccess(genSchema));
    }

View Full Code Here

                // the create() below will fail is this method gets called
                // more than once
                if(!new File(checkFileName).createNewFile()) {
                    throw new RuntimeException(checkFileName + " already exists!");
                }
                return new ResourceSchema(
                        Utils.getSchemaFromString("a:chararray,b:int"));
            } catch (ParseException e) {
                throw new IOException(e);
            }
        }

View Full Code Here

            ResourceFieldSchema stringfs = new ResourceFieldSchema();
            stringfs.setType(DataType.CHARARRAY);
            ResourceFieldSchema intfs = new ResourceFieldSchema();
            intfs.setType(DataType.INTEGER);
            
            ResourceSchema tupleSchema = new ResourceSchema();
            tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, intfs});
            ResourceFieldSchema tuplefs = new ResourceFieldSchema();
            tuplefs.setSchema(tupleSchema);
            tuplefs.setType(DataType.TUPLE);
            
            ResourceSchema bagSchema = new ResourceSchema();
            bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
            ResourceFieldSchema bagfs = new ResourceFieldSchema();
            bagfs.setSchema(bagSchema);
            bagfs.setType(DataType.BAG);
            
            t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes(), bagfs) : null);

View Full Code Here

                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName, 
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Util.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            Assert.assertTrue("Checking binstorage getSchema output", Schema.equals(

View Full Code Here

        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()
                                .getFileName(), new Job(conf));
            }
        }
    }

View Full Code Here

        }


        @Override
        public ResourceSchema getSchema(String location, Job job)
                throws IOException {
            return new ResourceSchema(schema);
        }

View Full Code Here

  @Override
  public ResourceSchema getSchema(String s, Job job) throws IOException {
    Schema newSchema =  new Schema();
    newSchema.add(new Schema.FieldSchema("timestamp", DataType.LONG));
    newSchema.add(new Schema.FieldSchema("map", DataType.MAP));
    return new ResourceSchema(newSchema);
  }

View Full Code Here

    public static ResourceSchema convertToResourceSchema(org.apache.hadoop.zebra.schema.Schema tSchema)
    throws IOException {
        if( tSchema == null )
            return null;


        ResourceSchema rSchema = new ResourceSchema();
        int fieldCount = tSchema.getNumColumns();
        ResourceFieldSchema[] rFields = new ResourceFieldSchema[fieldCount];
        for( int i = 0; i < fieldCount; i++ ) {
            org.apache.hadoop.zebra.schema.Schema.ColumnSchema cSchema = tSchema.getColumn( i );
            if( cSchema != null ) 
                rFields[i] = convertToResourceFieldSchema( cSchema );
            else
                rFields[i] = new ResourceFieldSchema();
        }
        rSchema.setFields( rFields );
        return rSchema;
    }

View Full Code Here

            if( cSchema.getType() == ColumnType.MAP ) {
              // Pig doesn't want any schema for a map field.
                field.setSchema( null );
            } else {
              org.apache.hadoop.zebra.schema.Schema fs = cSchema.getSchema();
              ResourceSchema rs = convertToResourceSchema( fs  );
              if( cSchema.getType() == ColumnType.COLLECTION ) {
                int count = fs.getNumColumns();
                if( count > 1 || ( count == 1 && fs.getColumn( 0 ).getType() != ColumnType.RECORD ) ) {
                  // Pig requires a record (tuple) as the schema for a BAG field.
                  ResourceFieldSchema fieldSchema = new ResourceFieldSchema();
                  fieldSchema.setSchema( rs );
                  fieldSchema.setType( ColumnType.RECORD.pigDataType() );
                  rs = new ResourceSchema();
                  rs.setFields( new ResourceFieldSchema[] { fieldSchema } );
                }
              }
                field.setSchema( rs );
            }
        }

View Full Code Here

    throws ExecException, SchemaMergeException, FrontendException {
        String [] aliases ={"f1", "f2"};
        byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
        Schema origSchema = TypeCheckingTestUtil.genFlatSchema(
                aliases,types);
        ResourceSchema rsSchema = new ResourceSchema(origSchema);
        assertEquals("num fields", aliases.length, rsSchema.getFields().length);
        ResourceSchema.ResourceFieldSchema[] fields = rsSchema.getFields();
        for (int i=0; i<fields.length; i++) {
            assertEquals(fields[i].getName(), aliases[i]);
            assertEquals(fields[i].getType(), types[i]);
        }
        Schema genSchema = Schema.getPigSchema(rsSchema);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.ResourceSchema

com.acme.io.JsonLoader

com.acme.io.JsonStorage

com.linkedin.pig.FastAvroStorageCommon

com.mongodb.hadoop.pig.BSONLoader

com.mongodb.hadoop.pig.BSONStorage

com.mongodb.hadoop.pig.BSONStorageTest

com.mongodb.hadoop.pig.JSONPigReplace

com.mongodb.hadoop.pig.MongoInsertStorage

com.mongodb.hadoop.pig.MongoLoader

com.mongodb.hadoop.pig.MongoStorage

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.