Package org.apache.pig.tools.pigstats

Source Code of org.apache.pig.tools.pigstats.ScriptState$FeatureVisitor

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.tools.pigstats;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NativeMapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFRJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeCogroup;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
import org.apache.pig.impl.logicalLayer.LOCogroup;
import org.apache.pig.impl.logicalLayer.LOCross;
import org.apache.pig.impl.logicalLayer.LODistinct;
import org.apache.pig.impl.logicalLayer.LOFilter;
import org.apache.pig.impl.logicalLayer.LOForEach;
import org.apache.pig.impl.logicalLayer.LOJoin;
import org.apache.pig.impl.logicalLayer.LOLimit;
import org.apache.pig.impl.logicalLayer.LONative;
import org.apache.pig.impl.logicalLayer.LOSort;
import org.apache.pig.impl.logicalLayer.LOSplit;
import org.apache.pig.impl.logicalLayer.LOStream;
import org.apache.pig.impl.logicalLayer.LOUnion;
import org.apache.pig.impl.logicalLayer.LOVisitor;
import org.apache.pig.impl.logicalLayer.LogicalOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.LOCogroup.GROUPTYPE;
import org.apache.pig.impl.logicalLayer.LOJoin.JOINTYPE;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.JarManager;
import org.apache.pig.newplan.Operator;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;

/**
* ScriptStates encapsulates settings for a Pig script that runs on a hadoop
* cluster. These settings are added to all MR jobs spawned by the script and
* in turn are persisted in the hadoop job xml. With the properties already in
* the job xml, users who want to know the relations between the script and MR
* jobs can derive them from the job xmls. 
*/
public class ScriptState {
   
    /**
     * Keys of Pig settings added in MR job
     */
    private enum PIG_PROPERTY {
        SCRIPT_ID           ("pig.script.id"),
        SCRIPT              ("pig.script"),      
        COMMAND_LINE        ("pig.command.line"),
        HADOOP_VERSION      ("pig.hadoop.version"),
        VERSION             ("pig.version"),
        INPUT_DIRS          ("pig.input.dirs"),
        MAP_OUTPUT_DIRS     ("pig.map.output.dirs"),
        REDUCE_OUTPUT_DIRS  ("pig.reduce.output.dirs"),
        JOB_PARENTS         ("pig.parent.jobid"),
        JOB_FEATURE         ("pig.job.feature"),
        SCRIPT_FEATURES     ("pig.script.features"),
        JOB_ALIAS           ("pig.alias");       
      
        private String displayStr;
       
        private PIG_PROPERTY(String s) {
            displayStr = s;
        }
       
        @Override
        public String toString() { return displayStr; }
    };
   
    /**
     * Features used in a Pig script
     */
    static enum PIG_FEATURE {
        UNKNOWN,
        MERGE_JION,
        REPLICATED_JOIN,
        SKEWED_JOIN,
        HASH_JOIN,
        COLLECTED_GROUP,
        MERGE_COGROUP,
        COGROUP,
        GROUP_BY,
        ORDER_BY,
        DISTINCT,
        STREAMING,
        SAMPLER,
        INDEXER,
        MULTI_QUERY,
        FILTER,
        MAP_ONLY,
        CROSS,
        LIMIT,
        UNION,
        COMBINER,
        NATIVE;
    };
   
    /**
     * Pig property that allows user to turn off the inclusion of settings
     * in the jobs
     */
    public static final String INSERT_ENABLED = "pig.script.info.enabled";
   
    /**
     * Restricts the size of Pig script stored in job xml
     */
    public static final int MAX_SCRIPT_SIZE = 10240;
   
    private static final Log LOG = LogFactory.getLog(ScriptState.class);

    private static ThreadLocal<ScriptState> tss = new ThreadLocal<ScriptState>();
      
    private String id;
   
    private String script;
    private String commandLine;
   
    private String pigVersion;
    private String hodoopVersion;
   
    private long scriptFeatures;
   
    private Map<MapReduceOper, String> featureMap = null;
    private Map<MapReduceOper, String> aliasMap = null;
   
    private List<PigProgressNotificationListener> listeners
            = new ArrayList<PigProgressNotificationListener>();
   
    public static ScriptState start(String commandLine) {
        ScriptState ss = new ScriptState(UUID.randomUUID().toString());
        ss.setCommandLine(commandLine);
        tss.set(ss);
        return ss;
    }
   
    private ScriptState(String id) {
        this.id = id;
        this.script = "";
    }

    public static ScriptState get() {
        if (tss.get() == null) {
            ScriptState.start("");
        }
        return tss.get();
    }          
      
    public void registerListener(PigProgressNotificationListener listener) {
        listeners.add(listener);
    }
       
    public void emitLaunchStartedNotification(int numJobsToLaunch) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.launchStartedNotification(numJobsToLaunch);
        }
    }
   
    public void emitJobsSubmittedNotification(int numJobsSubmitted) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.jobsSubmittedNotification(numJobsSubmitted);
        }       
    }
   
    public void emitJobStartedNotification(String assignedJobId) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.jobStartedNotification(assignedJobId);
        }
    }
   
    public void emitjobFinishedNotification(JobStats jobStats) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.jobFinishedNotification(jobStats);
        }
    }
   
    public void emitJobFailedNotification(JobStats jobStats) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.jobFailedNotification(jobStats);
        }
    }
   
    public void emitOutputCompletedNotification(OutputStats outputStats) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.outputCompletedNotification(outputStats);
        }
    }
   
    public void emitProgressUpdatedNotification(int progress) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.progressUpdatedNotification(progress);
        }
    }
   
    public void emitLaunchCompletedNotification(int numJobsSucceeded) {
        for (PigProgressNotificationListener listener: listeners) {
            listener.launchCompletedNotification(numJobsSucceeded);
        }
    }
   
    public void addSettingsToConf(MapReduceOper mro, Configuration conf) {
        LOG.info("Pig script settings are added to the job");
        conf.set(PIG_PROPERTY.HADOOP_VERSION.toString(), getHadoopVersion());
        conf.set(PIG_PROPERTY.VERSION.toString(), getPigVersion());
        conf.set(PIG_PROPERTY.SCRIPT_ID.toString(), id);
        conf.set(PIG_PROPERTY.SCRIPT.toString(), getScript());       
        conf.set(PIG_PROPERTY.COMMAND_LINE.toString(), getCommandLine());
       
        try {
            LinkedList<POStore> stores = PlanHelper.getStores(mro.mapPlan);
            ArrayList<String> outputDirs = new ArrayList<String>();
            for (POStore st: stores) { 
                outputDirs.add(st.getSFile().getFileName());
            }                
            conf.set(PIG_PROPERTY.MAP_OUTPUT_DIRS.toString(), LoadFunc.join(outputDirs, ","));
        } catch (VisitorException e) {
            LOG.warn("unable to get the map stores", e);
        }
        if (!mro.reducePlan.isEmpty()) {
            try {
                LinkedList<POStore> stores = PlanHelper.getStores(mro.reducePlan);
                ArrayList<String> outputDirs = new ArrayList<String>();
                for (POStore st: stores) { 
                    outputDirs.add(st.getSFile().getFileName());
                }                     
                conf.set(PIG_PROPERTY.REDUCE_OUTPUT_DIRS.toString(), LoadFunc.join(outputDirs, ","));
            } catch (VisitorException e) {
                LOG.warn("unable to get the reduce stores", e);
            }
        }       
        try {
            List<POLoad> lds = PlanHelper.getLoads(mro.mapPlan);
            ArrayList<String> inputDirs = new ArrayList<String>();
            if (lds != null && lds.size() > 0){
                for (POLoad ld : lds) {
                    inputDirs.add(ld.getLFile().getFileName());
                }              
                conf.set(PIG_PROPERTY.INPUT_DIRS.toString(), LoadFunc.join(inputDirs, ","));      
            }
        } catch (VisitorException e) {
            LOG.warn("unable to get the map loads", e);
        }

        setPigFeature(mro, conf);
       
        setJobParents(mro, conf);
    }
    public void setScript(File file) {           
        try {
            setScript(new BufferedReader(new FileReader(file)));
        } catch (FileNotFoundException e) {
            LOG.warn("unable to find the file", e);
        }
    }

    public void setScript(String script) {       
        if (script == null) return;
       
        // restrict the size of the script to be stored in job conf
        script = (script.length() > MAX_SCRIPT_SIZE) ? script.substring(0,
                MAX_SCRIPT_SIZE) : script;
       
        // XML parser cann't handle certain characters, including
        // the control character (&#1). Use Base64 encoding to
        // get around this problem       
        this.script = new String(Base64.encodeBase64(script.getBytes()));
    }
  
    public void setScriptFeatures(LogicalPlan plan) {
        BitSet bs = new BitSet();
        try {
            new LogicalPlanFeatureVisitor(plan, bs).visit();
        } catch (VisitorException e) {
            LOG.warn("unable to get script feature", e);
        }
        scriptFeatures = bitSetToLong(bs);       
       
        LOG.info("Pig features used in the script: "
                + featureLongToString(scriptFeatures));
    }
   
    public String getHadoopVersion() {
        if (hodoopVersion == null) {
            hodoopVersion = VersionInfo.getVersion();
        }
        return (hodoopVersion == null) ? "" : hodoopVersion;
    }
   
    public String getPigVersion() {
        if (pigVersion == null) {
            String findContainingJar = JarManager.findContainingJar(ScriptState.class);
            try {
                JarFile jar = new JarFile(findContainingJar);
                final Manifest manifest = jar.getManifest();
                final Map <String,Attributes> attrs = manifest.getEntries();
                Attributes attr = attrs.get("org/apache/pig");
                pigVersion = attr.getValue("Implementation-Version");
            } catch (Exception e) {
                LOG.warn("unable to read pigs manifest file");
            }
        }
        return (pigVersion == null) ? "" : pigVersion;
    }
       
    String getId() { return id; }
       
    private String getCommandLine() {
        return (commandLine == null) ? "" : commandLine;
    }
   
    private void setCommandLine(String commandLine) {
        this.commandLine = commandLine;
    }
   
    private String getScript() {
        return (script == null) ? "" : script;
    }
   
    private void setScript(BufferedReader reader) {
        StringBuilder sb = new StringBuilder();
        try {
            String line = reader.readLine();
            while (line != null) {
                if (line.length() > 0) {
                    sb.append(line).append("\n");
                }               
                line = reader.readLine();
            }           
        } catch (IOException e) {
            LOG.warn("unable to parse the script", e);
        }
        setScript(sb.toString());
    }
   
    private void setPigFeature(MapReduceOper mro, Configuration conf) {
        conf.set(PIG_PROPERTY.JOB_FEATURE.toString(), getPigFeature(mro));
        if (scriptFeatures != 0) {
            conf.set(PIG_PROPERTY.SCRIPT_FEATURES.toString(),
                    String.valueOf(scriptFeatures));
        }
        conf.set(PIG_PROPERTY.JOB_ALIAS.toString(), getAlias(mro));   
    }
   
    private void setJobParents(MapReduceOper mro, Configuration conf) {
        // PigStats maintains a job DAG with the job id being updated
        // upon available. Therefore, before a job is submitted, the ids
        // of its parent jobs are already available.
        JobGraph jg = PigStats.get().getJobGraph();
        JobStats js = null;
        Iterator<JobStats> iter = jg.iterator();
        while (iter.hasNext()) {
            JobStats job = iter.next();
            if (job.getName().equals(mro.getOperatorKey().toString())) {
                js = job;
                break;
            }
        }
        if (js != null) {
            List<Operator> preds = jg.getPredecessors(js);
            if (preds != null) {
                StringBuilder sb = new StringBuilder();
                for (Operator op : preds) {
                    JobStats job = (JobStats)op;
                    if (sb.length() > 0) sb.append(",");
                    sb.append(job.getJobId());
                }
                conf.set(PIG_PROPERTY.JOB_PARENTS.toString(), sb.toString());
            }
        }
    }
   
    String getScriptFeatures() {
        return featureLongToString(scriptFeatures);
    }
   
    String getAlias(MapReduceOper mro) {
        if (aliasMap == null) {
            aliasMap = new HashMap<MapReduceOper, String>();
        }
        String retStr = aliasMap.get(mro);
        if (retStr == null) {
            ArrayList<String> alias = new ArrayList<String>();
            try {
                new AliasVisitor(mro.mapPlan, alias).visit();
                new AliasVisitor(mro.reducePlan, alias).visit();
                if (!alias.isEmpty()) {
                    Collections.sort(alias);
                }             
            } catch (VisitorException e) {
                LOG.warn("unable to get alias", e);
            }
            retStr = LoadFunc.join(alias, ",");
            aliasMap.put(mro, retStr);
        }
        return retStr;
    }

    String getPigFeature(MapReduceOper mro) {
        if (featureMap == null) {
            featureMap = new HashMap<MapReduceOper, String>();
        }
       
        String retStr = featureMap.get(mro);
        if (retStr == null) {      
            BitSet feature = new BitSet();
            feature.clear();
            if (mro.isSkewedJoin()) {
                feature.set(PIG_FEATURE.SKEWED_JOIN.ordinal());
            }
            if (mro.isGlobalSort()) {
                feature.set(PIG_FEATURE.ORDER_BY.ordinal());
            }
            if (mro.isSampler()) {
                feature.set(PIG_FEATURE.SAMPLER.ordinal());
            }
            if (mro.isIndexer()) {
                feature.set(PIG_FEATURE.INDEXER.ordinal());
            }
            if (mro.isCogroup()) {
                feature.set(PIG_FEATURE.COGROUP.ordinal());
            }
            if (mro.isGroupBy()) {
                feature.set(PIG_FEATURE.GROUP_BY.ordinal());
            }
            if (mro.isRegularJoin()) {
                feature.set(PIG_FEATURE.HASH_JOIN.ordinal());
           
            if (mro.needsDistinctCombiner()) {
                feature.set(PIG_FEATURE.DISTINCT.ordinal());
            }
            if (!mro.combinePlan.isEmpty()) {
                feature.set(PIG_FEATURE.COMBINER.ordinal());
            }
            if (mro instanceof NativeMapReduceOper) {
                feature.set(PIG_FEATURE.NATIVE.ordinal());
            }
            else{// if it is NATIVE MR , don't explore its plans
                try {
                    new FeatureVisitor(mro.mapPlan, feature).visit();
                    if (mro.reducePlan.isEmpty()) {
                        feature.set(PIG_FEATURE.MAP_ONLY.ordinal());                   
                    } else {
                        new FeatureVisitor(mro.reducePlan, feature).visit();
                    }
                } catch (VisitorException e) {
                    LOG.warn("Feature visitor failed", e);
                }
            }
            StringBuilder sb = new StringBuilder();
            for (int i=feature.nextSetBit(0); i>=0; i=feature.nextSetBit(i+1)) {
                if (sb.length() > 0) sb.append(",");            
                sb.append(PIG_FEATURE.values()[i].name());
            }
            retStr = sb.toString();
            featureMap.put(mro, retStr);
        }
        return retStr;
    }   
   
    private long bitSetToLong(BitSet bs) {
        long ret = 0;
        for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i+1)) {
            ret |= (1L << i);
        }
        return ret;
    }
   
    private String featureLongToString(long l) {
        if (l == 0) return PIG_FEATURE.UNKNOWN.name();
       
        StringBuilder sb = new StringBuilder();
        for (int i=0; i<PIG_FEATURE.values().length; i++) {
            if (((l >> i) & 0x00000001) != 0) {
                if (sb.length() > 0) sb.append(",");
                sb.append(PIG_FEATURE.values()[i].name());
            }
        }
        return sb.toString();
    }
   
    private static class FeatureVisitor extends PhyPlanVisitor {
        private BitSet feature;
       
        public FeatureVisitor(PhysicalPlan plan, BitSet feature) {
            super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(
                    plan));
            this.feature = feature;
        }
       
        @Override
        public void visitFRJoin(POFRJoin join) throws VisitorException {
            feature.set(PIG_FEATURE.REPLICATED_JOIN.ordinal());
        }
       
        @Override
        public void visitMergeJoin(POMergeJoin join) throws VisitorException {
            feature.set(PIG_FEATURE.MERGE_JION.ordinal());
        }
       
        @Override
        public void visitMergeCoGroup(POMergeCogroup mergeCoGrp)
                throws VisitorException {
            feature.set(PIG_FEATURE.MERGE_COGROUP.ordinal());;
        }
       
        @Override
        public void visitCollectedGroup(POCollectedGroup mg)
                throws VisitorException {          
            feature.set(PIG_FEATURE.COLLECTED_GROUP.ordinal());
        }
       
        @Override
        public void visitDistinct(PODistinct distinct) throws VisitorException {
            feature.set(PIG_FEATURE.DISTINCT.ordinal());
        }
       
        @Override
        public void visitStream(POStream stream) throws VisitorException {
            feature.set(PIG_FEATURE.STREAMING.ordinal());
        }
       
        @Override
        public void visitSplit(POSplit split) throws VisitorException {
            feature.set(PIG_FEATURE.MULTI_QUERY.ordinal());
        }
       
        @Override
        public void visitDemux(PODemux demux) throws VisitorException {
            feature.set(PIG_FEATURE.MULTI_QUERY.ordinal());        
        }       
    }   
   
    public static class LogicalPlanFeatureVisitor extends LOVisitor {
       
        private BitSet feature;
       
        protected LogicalPlanFeatureVisitor(LogicalPlan plan, BitSet feature) {
            super(plan, new DepthFirstWalker<LogicalOperator,
                    LogicalPlan>(plan));           
            this.feature = feature;
        }
       
        @Override
        protected void visit(LOCogroup op) throws VisitorException {
            if (op.getGroupType() == GROUPTYPE.COLLECTED) {
                feature.set(PIG_FEATURE.COLLECTED_GROUP.ordinal());
            } else if (op.getGroupType() == GROUPTYPE.MERGE) {
                feature.set(PIG_FEATURE.MERGE_COGROUP.ordinal());               
            } else if (op.getGroupType() == GROUPTYPE.REGULAR) {
                if (op.getInputs().size() > 1) {
                    feature.set(PIG_FEATURE.COGROUP.ordinal());
                } else {
                    feature.set(PIG_FEATURE.GROUP_BY.ordinal());
                }
            }
        }
       
        @Override
        protected void visit(LOCross op) throws VisitorException {
            feature.set(PIG_FEATURE.CROSS.ordinal());
        }
       
        @Override
        protected void visit(LODistinct op) throws VisitorException {
            feature.set(PIG_FEATURE.DISTINCT.ordinal());
        }
       
        @Override
        protected void visit(LOFilter op) throws VisitorException {
            feature.set(PIG_FEATURE.FILTER.ordinal());
        }
       
        @Override
        protected void visit(LOForEach op) throws VisitorException {
           
        }
               
        @Override
        protected void visit(LOJoin op) throws VisitorException {
            if (op.getJoinType() == JOINTYPE.HASH) {
                feature.set(PIG_FEATURE.HASH_JOIN.ordinal());
            } else if (op.getJoinType() == JOINTYPE.MERGE) {
                feature.set(PIG_FEATURE.MERGE_JION.ordinal());
            } else if (op.getJoinType() == JOINTYPE.REPLICATED) {
                feature.set(PIG_FEATURE.REPLICATED_JOIN.ordinal());
            } else if (op.getJoinType() == JOINTYPE.SKEWED) {
                feature.set(PIG_FEATURE.SKEWED_JOIN.ordinal());
            }
        }
       
        @Override
        protected void visit(LOLimit op) throws VisitorException {
            feature.set(PIG_FEATURE.LIMIT.ordinal());
        }
       
        @Override
        protected void visit(LOSort op) throws VisitorException {
            feature.set(PIG_FEATURE.ORDER_BY.ordinal());
        }
       
        @Override
        protected void visit(LOStream op) throws VisitorException {
            feature.set(PIG_FEATURE.STREAMING.ordinal());
        }
       
        @Override
        protected void visit(LOSplit op) throws VisitorException {
           
        }
       
        @Override
        protected void visit(LOUnion op) throws VisitorException {
            feature.set(PIG_FEATURE.UNION.ordinal());
        }
       
        @Override
        protected void visit(LONative n) throws VisitorException {
            feature.set(PIG_FEATURE.NATIVE.ordinal());
        }

    }
   
    private static class AliasVisitor extends PhyPlanVisitor {
       
        private HashSet<String> aliasSet;
       
        private List<String> alias;
       
        public AliasVisitor(PhysicalPlan plan, List<String> alias) {
            super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(
                    plan));
            this.alias = alias;
            aliasSet = new HashSet<String>();
            if (!alias.isEmpty()) {
                for (String s : alias) aliasSet.add(s);
            }
        }
       
        @Override
        public void visitLoad(POLoad load) throws VisitorException {
            setAlias(load);
        }
       
        @Override
        public void visitFRJoin(POFRJoin join) throws VisitorException {
            setAlias(join);
        }
       
        @Override
        public void visitMergeJoin(POMergeJoin join) throws VisitorException {
            setAlias(join);
        }
       
        @Override
        public void visitMergeCoGroup(POMergeCogroup mergeCoGrp)
                throws VisitorException {
            setAlias(mergeCoGrp);
        }
       
        @Override
        public void visitCollectedGroup(POCollectedGroup mg)
                throws VisitorException {          
            setAlias(mg);
        }
       
        @Override
        public void visitDistinct(PODistinct distinct) throws VisitorException {
            setAlias(distinct);
        }
       
        @Override
        public void visitStream(POStream stream) throws VisitorException {
            setAlias(stream);
        }
       
        @Override
        public void visitFilter(POFilter fl) throws VisitorException {
            setAlias(fl);
        }
        
        @Override
        public void visitLocalRearrange(POLocalRearrange lr) throws VisitorException {
            setAlias(lr);
        }
       
        @Override
        public void visitPOForEach(POForEach nfe) throws VisitorException {
            setAlias(nfe);
        }
       
        @Override
        public void visitUnion(POUnion un) throws VisitorException {
            setAlias(un);
        }

        @Override
        public void visitSort(POSort sort) throws VisitorException {
            setAlias(sort);
        }
        @Override
        public void visitLimit(POLimit lim) throws VisitorException {
            setAlias(lim);
        }
       
        @Override
        public void visitSkewedJoin(POSkewedJoin sk) throws VisitorException {
            setAlias(sk);
        }
       
        private void setAlias(PhysicalOperator op) {
            String s = op.getAlias();
            if (s != null) {
                if (!aliasSet.contains(s)) {
                    alias.add(s);
                    aliasSet.add(s);
                }
            }
        }
    }
   
}
TOP

Related Classes of org.apache.pig.tools.pigstats.ScriptState$FeatureVisitor

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.