Package org.apache.pig.backend.local.executionengine.physicalLayer.counters

Examples of org.apache.pig.backend.local.executionengine.physicalLayer.counters.POCounter

364
365
366
367
368
369
370
371
372
373
374
        //store.setPc(pc);
        currentPlan.add(store);
        PhysicalOperator from = LogToPhyMap.get(loStore
                .getPlan().getPredecessors(loStore).get(0));
       
        POCounter counter = new POCounter(new OperatorKey(scope, nodeGen.getNextNodeId(scope)));
        currentPlan.add(counter);
        try {
            currentPlan.connect(from, counter);
            currentPlan.connect(counter, store);
        } catch (PlanException e) {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    private Map<String, Map<String, String>> accumulateLocalStats() {
        //The counter placed before a store in the local plan should be able to get the number of records
        for(PhysicalOperator op : php.getLeaves()) {
            Map<String, String> jobStats = new HashMap<String, String>();
            stats.put(op.toString(), jobStats);
            POCounter counter = (POCounter) php.getPredecessors(op).get(0);
            jobStats.put("PIG_STATS_LOCAL_OUTPUT_RECORDS", (Long.valueOf(counter.getCount())).toString());
            jobStats.put("PIG_STATS_LOCAL_BYTES_WRITTEN", (Long.valueOf((new File(((POStore)op).getSFile().getFileName())).length())).toString());
        }
        return stats;
    }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
    POCounter pcount;
   
    @Before
    public void setUp() throws Exception {
        st = GenPhyOp.topStoreOp();
        pcount = new POCounter(new OperatorKey("", (new Random()).nextLong()));
        fSpec = new FileSpec("file:/tmp/storeTest.txt",
                      new FuncSpec(PigStorage.class.getName(), new String[]{":"}));
        st.setSFile(fSpec);

        FileLocalizer.deleteTempFiles();
View Full Code Here
364
365
366
367
368
369
370
371
372
373
374
        //store.setPc(pc);
        currentPlan.add(store);
        PhysicalOperator from = logToPhyMap.get(loStore
                .getPlan().getPredecessors(loStore).get(0));
       
        POCounter counter = new POCounter(new OperatorKey(scope, nodeGen.getNextNodeId(scope)));
        currentPlan.add(counter);
        try {
            currentPlan.connect(from, counter);
            currentPlan.connect(counter, store);
        } catch (PlanException e) {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
    private Map<String, Map<String, String>> accumulateLocalStats() {
        //The counter placed before a store in the local plan should be able to get the number of records
        for(PhysicalOperator op : php.getLeaves()) {
            Map<String, String> jobStats = new HashMap<String, String>();
            stats.put(op.toString(), jobStats);
            POCounter counter = (POCounter) php.getPredecessors(op).get(0);
            jobStats.put("PIG_STATS_LOCAL_OUTPUT_RECORDS", (Long.valueOf(counter.getCount())).toString());
            String localFilePath=normalizeToLocalFilePath(((POStore)op).getSFile().getFileName());
            jobStats.put("PIG_STATS_LOCAL_BYTES_WRITTEN", (Long.valueOf(new File(localFilePath).length())).toString());
        }
        return stats;
    }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    private Map<String, Map<String, String>> accumulateLocalStats() {
        //The counter placed before a store in the local plan should be able to get the number of records
        for(PhysicalOperator op : php.getLeaves()) {
            Map<String, String> jobStats = new HashMap<String, String>();
            stats.put(op.toString(), jobStats);
            POCounter counter = (POCounter) php.getPredecessors(op).get(0);
            jobStats.put("PIG_STATS_LOCAL_OUTPUT_RECORDS", (Long.valueOf(counter.getCount())).toString());
            jobStats.put("PIG_STATS_LOCAL_BYTES_WRITTEN", (Long.valueOf((new File(((POStore)op).getSFile().getFileName())).length())).toString());
        }
        return stats;
    }
View Full Code Here
246
247
248
249
250
251
252
253
254
255
256
        //store.setPc(pc);
        currentPlan.add(store);
        PhysicalOperator from = LogToPhyMap.get(loStore
                .getPlan().getPredecessors(loStore).get(0));
       
        POCounter counter = new POCounter(new OperatorKey(scope, nodeGen.getNextNodeId(scope)));
        currentPlan.add(counter);
        try {
            currentPlan.connect(from, counter);
            currentPlan.connect(counter, store);
        } catch (PlanException e) {
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
    POCounter pcount;
   
    @Before
    public void setUp() throws Exception {
        st = GenPhyOp.topStoreOp();
        pcount = new POCounter(new OperatorKey("", (new Random()).nextLong()));
        fSpec = new FileSpec("file:/tmp/storeTest.txt",
                      new FuncSpec(PigStorage.class.getName(), new String[]{":"}));
        st.setSFile(fSpec);

        FileLocalizer.deleteTempFiles();
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    private Map<String, Map<String, String>> accumulateLocalStats() {
        //The counter placed before a store in the local plan should be able to get the number of records
        for(PhysicalOperator op : php.getLeaves()) {
            Map<String, String> jobStats = new HashMap<String, String>();
            stats.put(op.toString(), jobStats);
            POCounter counter = (POCounter) php.getPredecessors(op).get(0);
            jobStats.put("PIG_STATS_LOCAL_OUTPUT_RECORDS", (Long.valueOf(counter.getCount())).toString());
            jobStats.put("PIG_STATS_LOCAL_BYTES_WRITTEN", (Long.valueOf((new File(((POStore)op).getSFile().getFileName())).length())).toString());
        }
        return stats;
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.local.executionengine.physicalLayer.counters.POCounter

  • org.apache.pig.backend.local.executionengine.physicalLayer.LocalLogToPhyTranslationVisitor
  • org.apache.pig.test.TestStore
  • org.apache.pig.tools.pigstats.PigStats
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.