Package org.apache.pig.penny

Examples of org.apache.pig.penny.ParsedPigScript

31
32
33
34
35
36
37
38
39
40
41
42
43
44
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        for (String alias : parsedPigScript.aliases()) {
            monitorClasses.put(alias, new ClassWithArgs(DSMonitorAgent.class));
        }
        parsedPigScript.trace(DSCoordinator.class, monitorClasses);
       
    }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
       
        System.out.println("***** CRASH INVESTIGATOR STARTING ...");
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);

        Map<String, Integer> lowerBounds = new HashMap<String, Integer>();            // lower bound on tuple num for each logical location
        Map<PhysicalLocation, Tuple> latestTuples = null;
        for (int skipSize = 100; skipSize > 0; skipSize /= 10) {
            System.out.println("***** TRYING SCRIPT ...");
           
            Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
            for (String alias : parsedPigScript.aliases()) {
                int lowerBound = (lowerBounds.containsKey(alias))? lowerBounds.get(alias) : 0;
                monitorClasses.put(alias, new ClassWithArgs(CIMonitorAgent.class, lowerBound, skipSize));
            }
            latestTuples = (Map<PhysicalLocation, Tuple>) parsedPigScript.trace(CICoordinator.class, monitorClasses);
           
            // update lower bounds
            lowerBounds.clear();
            for (Entry<PhysicalLocation, Tuple> entry : latestTuples.entrySet()) {
                String alias = entry.getKey().logId();
                int tupleNum = (Integer) entry.getValue().get(0);
                if (lowerBounds.containsKey(alias)) lowerBounds.put(alias, Math.min(lowerBounds.get(alias), tupleNum));
                else lowerBounds.put(alias, tupleNum);
            }
        }
       
        Map<String, Set<Tuple>> crashCulprits = new HashMap<String, Set<Tuple>>();
        for (Entry<PhysicalLocation, Tuple> entry : latestTuples.entrySet()) {
            String alias = entry.getKey().logId();
            Tuple t = (Tuple) entry.getValue().get(1);
            if (!crashCulprits.containsKey(alias)) crashCulprits.put(alias, new HashSet<Tuple>());
            crashCulprits.get(alias).add(t);
        }
       
        System.out.println("\n\n***** CRASH SUSPECTS:");
        for (String alias : parsedPigScript.aliases()) {
            if (crashCulprits.containsKey(alias)) {
                System.out.println("\nAlias " + alias + ":");
                for (Tuple t : crashCulprits.get(alias)) {
                    System.out.println("\t" + t);
                }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        String triggerAlias = args[1];                            // script alias at which to trigger forward tracing
        int triggerField = Integer.parseInt(args[2]);            // field number to use for triggering forward tracing
        String triggerValue = args[3];                            // data value to use for triggering forward tracing (for now, use a String -- in general should be any Object)
       
        if (!parsedPigScript.aliases().contains(triggerAlias)) throw new IllegalArgumentException("Invalid trigger alias.");

        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        for (String alias : parsedPigScript.aliases()) {
            if (alias.equals(triggerAlias)) {
                monitorClasses.put(alias, new ClassWithArgs(FTMonitorAgent.class, triggerField, triggerValue));
            } else {
                monitorClasses.put(alias, new ClassWithArgs(FTMonitorAgent.class));
            }
        }
        parsedPigScript.trace(FTCoordinator.class, monitorClasses);
       
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        for (String alias : parsedPigScript.aliases()) {
            if (parsedPigScript.operator(alias).equals("LOLoad")) {
                monitorClasses.put(alias, new ClassWithArgs(TRMonitorAgent.class));
                break;            // only put MA at (first) load
            }
        }
        parsedPigScript.trace(TRCoordinator.class, monitorClasses);
       
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
   
        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        for (String alias : parsedPigScript.aliases()) {
            monitorClasses.put(alias, new ClassWithArgs(LPMonitorAgent.class, parsedPigScript.operator(alias).equals("LOLoad")));
        }
        parsedPigScript.trace(LPCoordinator.class, monitorClasses);
       
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        String alias = args[1];                                // which alias to check
        int minCard = Integer.parseInt(args[2]);            // minimum cardinality accepted
       
        if (!parsedPigScript.aliases().contains(alias)) throw new IllegalArgumentException("Invalid alias.");

        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        monitorClasses.put(alias, new ClassWithArgs(TIMonitorAgent.class));
        parsedPigScript.trace(new ClassWithArgs(TICoordinator.class, minCard), monitorClasses);
    }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        if (!parsedPigScript.isChain()) throw new RuntimeException("Overhead profiling currently only supports linear chain scripts (no joins, splits, etc.).");

        System.out.println("***** OVERHEAD PROFILER STARTING ...");
       
        Map<Integer, Long> runningTimes = new HashMap<Integer, Long>();                // running time of a script prefix ending at line N

        List<String> runs = new ArrayList<String>();
        String loadAlias = getLoadAlias(parsedPigScript);
        for (String alias = loadAlias; alias != null; alias = parsedPigScript.outEdge(alias)) {
            runs.add(alias);
        }
        runs.add(null);        // whole thing
       
        for (int prefix = 0; prefix < runs.size(); prefix++) {
            System.out.println("*** LAUNCHING PROFILE RUN " + (prefix+1) + " of " + runs.size() + " ...");
       
            Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
            if (runs.get(prefix) != null) monitorClasses.put(runs.get(prefix), new ClassWithArgs(OPMonitorAgent.class));
           
            long totalRunningTime = 0;
            for (int i = 0; i < NUM_TRIALS; i++) {
                System.out.println("* TRIAL " + (i+1) + " of " + NUM_TRIALS + " ...");
               
                long startTime = System.currentTimeMillis();
                parsedPigScript.truncateAndTrace(OPCoordinator.class, monitorClasses, runs.get(prefix));
                totalRunningTime += System.currentTimeMillis() - startTime;
            }
            long avgRunningTime = totalRunningTime/NUM_TRIALS;
            runningTimes.put(prefix, avgRunningTime);
        }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
        } else {
            System.out.println("Adding instrumentation");
        }
        PennyServer igServer= new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = igServer.parse(pigScriptFilename);

        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        if (!skipInstrumentation) {
            for (String alias: parsedPigScript.aliases()) {
                monitorClasses.put(alias, new ClassWithArgs(NOPMonitorAgent.class));
            }
        }
        parsedPigScript.trace(NOPCoordinator.class, monitorClasses);
    }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        String alias = args[1];                                // which alias to create histogram for
        int fieldNo = Integer.parseInt(args[2]);            // which field to create histogram for
        int min = Integer.parseInt(args[3]);                // min field value
        int max = Integer.parseInt(args[4]);                // max field value
        int bucketSize = Integer.parseInt(args[5]);            // histogram bucket size
       
        if (!parsedPigScript.aliases().contains(alias)) throw new IllegalArgumentException("No such alias.");
       
        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        monitorClasses.put(alias, new ClassWithArgs(DHMonitorAgent.class, fieldNo, min, max, bucketSize));
        TreeMap<Integer, Integer> histogram = (TreeMap<Integer, Integer>) parsedPigScript.trace(DHCoordinator.class, monitorClasses);
        System.out.println("Histogram: " + histogram);
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
   
    public static void main(String[] args) throws Exception {
       
        PennyServer pennyServer = new PennyServer();
        String pigScriptFilename = args[0];
        ParsedPigScript parsedPigScript = pennyServer.parse(pigScriptFilename);
       
        String alias = args[1];                                // which alias to check
        int fieldNo = Integer.parseInt(args[2]);            // which field to check for nulls
       
        if (!parsedPigScript.aliases().contains(alias)) throw new IllegalArgumentException("Invalid alias.");

        Map<String, ClassWithArgs> monitorClasses = new HashMap<String, ClassWithArgs>();
        monitorClasses.put(alias, new ClassWithArgs(RIMonitorAgent.class, fieldNo));
        parsedPigScript.trace(RICoordinator.class, monitorClasses);
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.penny.ParsedPigScript

  • org.apache.pig.FuncSpec
  • org.apache.pig.impl.io.FileSpec
  • org.apache.pig.newplan.logical.expression.LogicalExpression
  • org.apache.pig.newplan.logical.expression.LogicalExpressionPlan
  • org.apache.pig.newplan.logical.expression.ProjectExpression
  • org.apache.pig.newplan.logical.expression.UserFuncExpression
  • org.apache.pig.newplan.logical.relational.LOForEach
  • org.apache.pig.newplan.logical.relational.LOGenerate
  • org.apache.pig.newplan.logical.relational.LogicalPlan
  • org.apache.pig.newplan.logical.relational.LogicalRelationalOperator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.