Package org.apache.pig.impl

Examples of org.apache.pig.impl.PigContext.connect()

139
140
141
142
143
144
145
146
147
148
149
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConfInternal.get()));
            try {
                pc.connect();
           
                ld.setPc(pc);
                Tuple dummyTuple = null;
                for(Result res=ld.getNext(dummyTuple);res.returnStatus!=POStatus.STATUS_EOP;res=ld.getNext(dummyTuple)){
                    Tuple tup = (Tuple) res.result;
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
        Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
        Map<OperatorKey, LogicalOperator> opTable = new HashMap<OperatorKey, LogicalOperator>() ;
        Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>() ;
        Map<String, String> fileNameMap = new HashMap<String, String>();
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties()) ;
        pigContext.connect();
       
        String tempFile = this.prepareTempFile() ;
       
        // Start the real parsing job
        {
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            try {
                pc.connect();
           
                ld.setPc(pc);
                Tuple dummyTuple = null;
                for(Result res=ld.getNext(dummyTuple);res.returnStatus!=POStatus.STATUS_EOP;res=ld.getNext(dummyTuple)){
                    Tuple tup = (Tuple) res.result;
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
            lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            LogicalPlan lp = lpt.buildPlan("C = cogroup A by id, B by id, D by id using 'merge';");
            assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPhysicalPlan(lp, pc);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
            assertTrue(phyOp instanceof POMergeCogroup);

            lp = lpt.buildPlan("store C into 'out';");
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
        lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("C = cogroup E by A.id, B by id, D by id using 'merge';");
        assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
            Util.buildPhysicalPlan(lp, pc);  
        }catch (LogicalToPhysicalTranslatorException e){
            assertEquals(1103,e.getErrorCode());
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
        lpt.buildPlan("D = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("C = cogroup A by id inner, B by id, D by id inner using 'merge';");
        assertEquals(LOCogroup.GROUPTYPE.MERGE, ((LOCogroup)lp.getLeaves().get(0)).getGroupType());

        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        boolean exceptionCaught = false;
        try{
            Util.buildPhysicalPlan(lp, pc);  
        }catch (LogicalToPhysicalTranslatorException e){
            exceptionCaught = true;
View Full Code Here
808
809
810
811
812
813
814
815
816
817
818
    public LogicalPlan buildPlan(String query, ClassLoader cldr) {
        LogicalPlanBuilder.classloader = cldr;
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
        try {

            pigContext.connect();
           
            LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //


            LogicalPlan lp = builder.parse("Test-Plan-Builder",
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
     */
    @Test
    public void testGroupConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by 1;");
       
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
     */
    @Test
    public void testGroupNonConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
       
View Full Code Here
463
464
465
466
467
468
469
470
471
472
473
        tester.buildPlan("A = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("B = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("C = join A by $0, B by $0 using \"merge\" parallel 50;");
        LogicalPlan lp = tester.buildPlan("store C into 'out';");
  PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
  MROperPlan mro = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);
        Assert.assertEquals(1,mro.getRoots().get(0).getRequestedParallelism());
    }

    @Test
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.