Package org.apache.pig.impl

Examples of org.apache.pig.impl.PigContext.connect()

252
253
254
255
256
257
258
259
260
261
262
    private LogicalPlan buildPlanThrowExceptionOnError (String query, ClassLoader cldr) throws IOException, ParseException {

        LogicalPlanBuilder.classloader = LogicalPlanTester.class.getClassLoader() ;
        PigContext pigContext = new PigContext(ExecType.MAPREDUCE, new Properties());
        try {
            pigContext.connect();
        } catch (ExecException e1) {
            fail(e1.getClass().getName() + ": " + e1.getMessage() + " -- " + query);
        }
        LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
                continue;
            }

            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile, false);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            pc.connect();
            ld.setPc(pc);
            POLocalRearrange lr = LRs[i];
            lr.setInputs(Arrays.asList((PhysicalOperator)ld));
            Map<Tuple, List<Tuple>> replicate = new HashMap<Tuple, List<Tuple>>(1000);
            log.debug("Completed setup. Trying to build replication hash table");
View Full Code Here
772
773
774
775
776
777
778
779
780
781
782
    public LogicalPlan buildPlan(String query, ClassLoader cldr) {
        LogicalPlanBuilder.classloader = cldr;
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
        try {

            pigContext.connect();
           
            LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //


            LogicalPlan lp = builder.parse("Test-Plan-Builder",
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
   
    @Test
    public void testLocalModeInputPositive() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.LOCAL, new Properties()) ;
        ctx.connect() ;
       
        String inputfile = generateTempFile().getAbsolutePath() ;
        String outputfile = generateNonExistenceTempFile().getAbsolutePath() ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getFs()) ;       
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
      
    @Test
    public void testLocalModeNegative2() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.LOCAL, new Properties()) ;
        ctx.connect() ;
       
        String inputfile = generateTempFile().getAbsolutePath() ;
        String outputfile = generateTempFile().getAbsolutePath() ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;       
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
       
    @Test
    public void testMapReduceModeInputPositive() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties()) ;      
        ctx.connect() ;
       
        String inputfile = createHadoopTempFile(ctx) ;
        String outputfile = createHadoopNonExistenceTempFile(ctx) ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;                    
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      
    @Test
    public void testMapReduceModeInputNegative2() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties()) ;      
        ctx.connect() ;
       
        String inputfile = createHadoopTempFile(ctx) ;
        String outputfile = createHadoopTempFile(ctx) ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;                    
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
        tester.buildPlan("A = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("B = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("C = join A by $0, B by $0 using \"merge\" parallel 50;");
        LogicalPlan lp = tester.buildPlan("store C into 'out';");
  PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
  MROperPlan mro = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);
        Assert.assertEquals(1,mro.getRoots().get(0).getRequestedParallelism());
    }

    @Test
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L),
                    replFile, false);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,
                    ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            pc.connect();
            ld.setPc(pc);
            // We use LocalRearrange Operator to seperate Key and Values
            // eg. ( a, b, c ) would generate a, ( a, b, c )
            // And we use 'a' as the key to the HashMap
            // The rest '( a, b, c )' is added to HashMap as value
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile, false);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            try {
                pc.connect();
           
                ld.setPc(pc);
                Tuple dummyTuple = null;
                for(Result res=ld.getNext(dummyTuple);res.returnStatus!=POStatus.STATUS_EOP;res=ld.getNext(dummyTuple)){
                    Tuple tup = (Tuple) res.result;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.