Examples of org.apache.pig.newplan.logical.relational.LOLoad

Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOLoad

org.apache.pig.newplan.logical.relational.LOLoad

    public void testRelationalEqualityOneNullOneNotNullSchema() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad load1 = newLOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
                conf);
        lp.add(load1);


        // Test that one with schema and one without breaks equality
        LOLoad load9 = newLOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp, conf);
        lp.add(load9);


        assertFalse(load1.isEqual(load9));
    }

View Full Code Here

    public void testJoinDifferentJoinTypes() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema jaschema1 = new LogicalSchema();
        jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema1, lp, conf);
        lp.add(A1);


        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad B1 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
        lp.add(B1);


        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, 0, 0, null);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, 1, 0, null);
        MultiMap<Integer, LogicalExpressionPlan> mm1 =
                new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);


        // A = load
        LogicalSchema jaschema2 = new LogicalSchema();
        jaschema2.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A2 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema2, lp, conf);
        lp.add(A2);


        // B = load
        LogicalSchema jbschema2 = new LogicalSchema();
        jbschema2.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad B2 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema2, lp, conf);
        lp.add(B2);


        // C = join
        LogicalSchema jcschema2 = new LogicalSchema();

View Full Code Here

    public void testJoinDifferentInner() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema jaschema1 = new LogicalSchema();
        jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema1, lp, conf);
        lp.add(A1);


        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad B1 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
        lp.add(B1);


        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, 0, 0, null);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, 1, 0, null);
        MultiMap<Integer, LogicalExpressionPlan> mm1 =
                new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);


        // Test different inner status
        // A = load
        LogicalSchema jaschema3 = new LogicalSchema();
        jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A3 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema3, lp, conf);
        lp.add(A3);


        // B = load
        LogicalSchema jbschema3 = new LogicalSchema();
        jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad B3 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema3, lp, conf);
        lp.add(B3);


        // C = join
        LogicalSchema jcschema3 = new LogicalSchema();

View Full Code Here

    public void testJoinDifferentNumInputs() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema jaschema1 = new LogicalSchema();
        jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema1, lp, conf);
        lp.add(A1);


        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad B1 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
        lp.add(B1);


        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, 0, 0, null);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, 1, 0, null);
        MultiMap<Integer, LogicalExpressionPlan> mm1 =
                new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);


        // A = load
        LogicalSchema jaschema5 = new LogicalSchema();
        jaschema5.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A5 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema5, lp, conf);
        lp.add(A5);


        // B = load
        LogicalSchema jbschema5 = new LogicalSchema();
        jbschema5.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad B5 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema5, lp, conf);
        lp.add(B5);


        // Beta = load
        LogicalSchema jbetaschema5 = new LogicalSchema();
        jbetaschema5.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LOLoad Beta5 = newLOLoad(new FileSpec("/ghi",
                new FuncSpec("PigStorage", "\t")), jbetaschema5, lp, conf);
        lp.add(Beta5);


        // C = join
        LogicalSchema jcschema5 = new LogicalSchema();

View Full Code Here


        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A6 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema6, lp, conf);
        lp.add(A6);


        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "z", null, DataType.LONG));
        LOLoad B6 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema6, lp, conf);
        lp.add(B6);


        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 =
                new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] { true, true });
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);


        LogicalSchema jaschema7 = new LogicalSchema();
        jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A7 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema7, lp, conf);
        lp.add(A7);


        // B = load
        LogicalSchema jbschema7 = new LogicalSchema();
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
                "z", null, DataType.LONG));
        LOLoad B7 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema7, lp, conf);
        lp.add(B7);


        // C = join
        LogicalSchema jcschema7 = new LogicalSchema();

View Full Code Here


        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A6 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema6, lp, conf);
        lp.add(A6);


        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "z", null, DataType.LONG));
        LOLoad B6 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema6, lp, conf);
        lp.add(B6);


        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 =
                new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] { true, true });
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);


        // Test different different number of join keys
        LogicalSchema jaschema8 = new LogicalSchema();
        jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A8 = newLOLoad(new FileSpec("/abc",
                new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                                "x", "y" })), jaschema8, lp, conf);
        lp.add(A8);


        // B = load
        LogicalSchema jbschema8 = new LogicalSchema();
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
                "z", null, DataType.LONG));
        LOLoad B8 = newLOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), jbschema8, lp, conf);
        lp.add(B8);


        // C = join
        LogicalSchema jcschema8 = new LogicalSchema();

View Full Code Here

    public void testRelationalSameOpDifferentPreds() throws FrontendException {
        LogicalPlan lp1 = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        LOLoad A1 = newLOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp1,
                conf);
        lp1.add(A1);


        LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
        ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
        ConstantExpression fc1 = new ConstantExpression(fp1,
                new Integer(0));
        new EqualExpression(fp1, fy1, fc1);
        LOFilter D1 = new LOFilter(lp1, fp1);
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
        D1.neverUseForRealSetSchema(cschema);
        lp1.add(D1);
        lp1.connect(A1, D1);


        LogicalPlan lp2 = new LogicalPlan();
        LOLoad A2 = newLOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp2,
                conf);
        lp2.add(A2);


        LogicalExpressionPlan fp2 = new LogicalExpressionPlan();

View Full Code Here

        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        schema.addField(new LogicalSchema.LogicalFieldSchema("age", null, DataType.INTEGER));    
        schema.getField(0).uid = 1;
        schema.getField(1).uid = 2;
        schema.getField(2).uid = 3;
        LogicalRelationalOperator l1 = new LOLoad(schema, plan);
        l1.setAlias("A");
        plan.add(l1);


        schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.INTEGER));
        schema.addField(new LogicalSchema.LogicalFieldSchema("dept", null, DataType.INTEGER));
        schema.addField(new LogicalSchema.LogicalFieldSchema("salary", null, DataType.FLOAT));    
        schema.getField(0).uid = 4;
        schema.getField(1).uid = 5;
        schema.getField(2).uid = 6;
        LogicalRelationalOperator l2 = new LOLoad(schema, plan);
        l2.setAlias("B");
        plan.add(l2);
        
        MultiMap<Integer, LogicalExpressionPlan> joinPlans = new MultiMap<Integer, LogicalExpressionPlan>();
        
        LogicalRelationalOperator j1 = new LOJoin(plan, joinPlans, LOJoin.JOINTYPE.HASH, new boolean[]{true, true});

View Full Code Here

    public void testLoaderSignature() throws Exception {
      String query = "a = load '1.txt' using org.apache.pig.test.PigStorageWithSchema() as (a0:int, a1:int);" +
                     "store a into 'output';";
        LogicalPlan plan = buildPlan( query );
        Operator op = plan.getSinks().get(0);
        LOLoad load = (LOLoad)plan.getPredecessors(op).get(0);
        // the signature is now a unique string of the format "{alias}_{scope id}-{id}" example: "a_12-0"
        String udfContextSignature = ((PigStorageWithSchema)(load).getLoadFunc()).getUDFContextSignature();
        Assert.assertTrue(udfContextSignature, udfContextSignature.matches("a_[0-9]*-[0-9]*"));


        query = " b = load '1.txt' using org.apache.pig.test.PigStorageWithSchema();" +

View Full Code Here

        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = newLOLoad( filespec1, null, plan,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;
        LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface)PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null) ;


        plan.add(load) ;
        plan.add(store) ;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOLoad

org.apache.hadoop.mapreduce.Job

org.apache.pig.impl.logicalLayer.schema.Schema

org.apache.pig.newplan.logical.rules.LimitOptimizer$OptimizeLimitTransformer

org.apache.pig.newplan.logical.rules.LoadTypeCastInserter

org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer

org.apache.pig.newplan.logical.rules.PredicatePushdownOptimizer

org.apache.pig.newplan.logical.rules.TypeCastInserter

org.apache.pig.parser.LogicalPlanBuilder

org.apache.pig.ResourceSchema

org.apache.pig.test.TestBlackAndWhitelistValidator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.