Examples of org.apache.pig.impl.logicalLayer.schema.Schema

Package org.apache.pig.impl.logicalLayer.schema

Examples of org.apache.pig.impl.logicalLayer.schema.Schema

org.apache.pig.impl.logicalLayer.schema.Schema
The Schema class encapsulates the notion of a schema for a relational operator. A schema is a list of columns that describe the output of a relational operator. Each column in the relation is represented as a FieldSchema, a static class inside the Schema. A column by definition has an alias, a type and a possible schema (if the column is a bag or a tuple). In addition, each column in the schema has a unique auto generated name used for tracking the lineage of the column in a sequence of statements. The lineage of the column is tracked using a map of the predecessors' columns to the operators that generate the predecessor columns. The predecessor columns are the columns required in order to generate the column under consideration. Similarly, a reverse lookup of operators that generate the predecessor column to the predecessor column is maintained.

        assertNotNull(Utils.getSchemaFromString(Utils.getSchemaFromString("name:bytearray,links:{(missing:chararray)}").toString()));
    }


    @Test
    public void testSchemaSerializationPlusBase64() throws Exception {
        Schema schemaFromString = Utils.getSchemaFromString("name:bytearray,links:{(missing:chararray)}");
        Schema schemaSaved = IOUtils.deserializeFromBase64(IOUtils.serializeToBase64(schemaFromString));
        assertEquals(schemaFromString.toString(), schemaSaved.toString());
    }

View Full Code Here

    }


    @Test
    public void testProjection() throws Exception {
        String schemaString = "ES_PARENT: {(parent_name: chararray,parent_value: chararray)}";
        Schema schema = Utils.getSchemaFromString(schemaString);
        System.out.println(PigUtils.asProjection(schema, new Properties()));
    }

View Full Code Here

        return output;
    }


    public Schema outputSchema(Schema input) {
        try {
            return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), DataType.TUPLE));
        } catch (Exception e) {
            return null;
        }
    }

View Full Code Here

        
    @Test
    public void testFRJoinSch1() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by $0, B by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by $0, B by $0;");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertEquals(true, shjSch.equals(frjSch));

View Full Code Here

    
    @Test
    public void testFRJoinSch2() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "';");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by $0, B by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by $0, B by $0;");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertTrue(shjSch == null);

View Full Code Here

    @Test
    public void testFRJoinSch3() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("C = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("D");
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0;");
        shjSch = pigServer.dumpSchema("D");
        Assert.assertEquals(true, shjSch.equals(frjSch));

View Full Code Here

    @Test
    public void testFRJoinSch4() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("C = LOAD '" + INPUT_FILE + "';");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("D");
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0;");
        shjSch = pigServer.dumpSchema("D");
        Assert.assertTrue(shjSch == null);

View Full Code Here

    
    @Test
    public void testFRJoinSch5() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1) using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1);");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertEquals(true, shjSch.equals(frjSch));

View Full Code Here

    
    @Test
    public void testFRJoinSch6() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "';");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1) using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1);");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertTrue(shjSch == null);

View Full Code Here

    public void testPigStorageSchema() throws Exception {
        pigContext.connect();
        String query = "a = LOAD 'originput' using org.apache.pig.piggybank.storage.PigStorageSchema() " +
        "as (f1:chararray, f2:int);";
        pig.registerQuery(query);
        Schema origSchema = pig.dumpSchema("a");
        pig.registerQuery("STORE a into 'aout' using org.apache.pig.piggybank.storage.PigStorageSchema();");
        
        // aout now has a schema. 


        // Verify that loading a-out with no given schema produces 
        // the original schema.
        
        pig.registerQuery("b = LOAD 'aout' using org.apache.pig.piggybank.storage.PigStorageSchema();");
        Schema genSchema = pig.dumpSchema("b");
        Assert.assertTrue("generated schema equals original" ,
                Schema.equals(genSchema, origSchema, true, false));
        
        // Verify that giving our own schema works
        String [] aliases ={"foo", "bar"};
        byte[] types = {DataType.INTEGER, DataType.LONG};
        Schema newSchema = TypeCheckingTestUtil.genFlatSchema(
                aliases,types);
        pig.registerQuery("c = LOAD 'aout' using org.apache.pig.piggybank.storage.PigStorageSchema() "+
        "as (foo:int, bar:long);");
        Schema newGenSchema = pig.dumpSchema("c");
        Assert.assertTrue("explicit schema overrides metadata",
                Schema.equals(newSchema, newGenSchema, true, false));
        
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.impl.logicalLayer.schema.Schema

com.manning.hip.ch11.PigGeolocationUDF

org.apache.howl.pig.HowlStorer

org.apache.howl.pig.TestHowlLoader

org.apache.oozie.action.hadoop.UDFTester

org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc

org.apache.pig.builtin.ABS

org.apache.pig.builtin.AddDuration

org.apache.pig.builtin.CONCAT

org.apache.pig.builtin.DaysBetween

org.apache.pig.builtin.ENDSWITH

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.