Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.BufferedPositionedInputStream

94
95
96
97
98
99
100
101
            // has actually consumed
            end = Long.MAX_VALUE;
        } else {
            is = fsis;
        }
        loader.bindTo(file.toString(), new BufferedPositionedInputStream(is,
                start), start, end);
    }
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
        pigServer.deleteFile(output);
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
       
        InputStream op = FileLocalizer.open(output, pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE);
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
        pigServer.deleteFile(output);
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
       
        InputStream op = FileLocalizer.open(output, pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE);
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }
View Full Code Here
432
433
434
435
436
437
438
439
440
441
442
        pigServer.deleteFile(output);
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
       
        InputStream op = FileLocalizer.open(output, pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE);
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }
View Full Code Here
494
495
496
497
498
499
500
501
502
503
504
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
       
        InputStream op = FileLocalizer.open(output+"/bar",
                                            pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE);
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }
View Full Code Here
557
558
559
560
561
562
563
564
565
566
567
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
       
        InputStream op = FileLocalizer.open(output+"/bar",
                                            pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE);
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }
View Full Code Here
620
621
622
623
624
625
626
627
628
629
630
        pigServer.store("OP", output, PigStorage.class.getName() + "(',')");
       
        InputStream op = FileLocalizer.open(output+"/foobar",
                                            pigServer.getPigContext());
        PigStorage ps = new PigStorage(",");
        ps.bindTo("", new BufferedPositionedInputStream(op), 0, Long.MAX_VALUE);
        List<Tuple> outputs = new ArrayList<Tuple>();
        Tuple t;
        while ((t = ps.getNext()) != null) {
            outputs.add(t);
        }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            DataBag quantilesList;
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t==null) throw new RuntimeException("Empty samples file");
            // the Quantiles file has a tuple as under:
            // (numQuantiles, bag of samples)
            // numQuantiles here is the reduce parallelism
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
        String filename = TestHelper.createTempFile(data, " ");
        CommonLogLoader commonLogLoader = new CommonLogLoader();
        PigServer pigServer = new PigServer(LOCAL);
       
        InputStream inputStream = FileLocalizer.open(filename, pigServer.getPigContext());
        commonLogLoader.bindTo(filename, new BufferedPositionedInputStream(inputStream), 0, Long.MAX_VALUE);

        int tupleCount = 0;

        while (true) {
            Tuple tuple = commonLogLoader.getNext();
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
        );
       
        String filename = input.getAbsolutePath();
        */
        InputStream inputStream = FileLocalizer.open(filename, pigServer.getPigContext());
        dummyRegExLoader.bindTo(filename, new BufferedPositionedInputStream(inputStream), 0, Long.MAX_VALUE);
        ArrayList<DataByteArray[]> expected = TestHelper.getExpected(data, pattern);
        int tupleCount = 0;
        while (true) {
            Tuple tuple = dummyRegExLoader.getNext();
            if (tuple == null)
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.BufferedPositionedInputStream

  • org.apache.pig.backend.executionengine.PigSlice
  • org.apache.pig.backend.hadoop.executionengine.HJob
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SortPartitioner
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin
  • org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil
  • org.apache.pig.backend.local.executionengine.LocalJob
  • org.apache.pig.backend.local.executionengine.POLoad
  • org.apache.pig.builtin.BinStorage
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.