Package org.apache.pig.impl.eval

Examples of org.apache.pig.impl.eval.EvalSpec$UserComparator


            }

            groupSpecs.add((ProjectSpec) args);*/
          List<Integer> cols = new ArrayList<Integer>();
          for(Iterator<EvalSpec> it = ((GenerateSpec)(op.getSpecs().get(i))).getSpecs().iterator(); it.hasNext();) {
            EvalSpec arg = it.next();
            if(arg instanceof ProjectSpec) {
              int numGroupColsThisInput = ((ProjectSpec) arg).numCols();
                    if (numGroupCols == -1) {
                        numGroupCols = numGroupColsThisInput;
                    } else {
View Full Code Here


        TupleSchema inputSchema = inputOp.outputSchema();
        DataBag inputConstraints = BagFactory.getInstance().newDefaultBag();
        DataBag outputData = derivedData.get(op);
        DataBag inputData = derivedData.get(inputOp);

        EvalSpec spec = op.getSpec();
        if (spec instanceof ProjectSpec) {
            ProjectSpec pSpec = (ProjectSpec) spec;
            List<Integer> cols = pSpec.getCols();

            if (outputConstraints.cardinality() > 0) {   // there's one or more output constraints; propagate them backwards through the projection
View Full Code Here

      //if any member of the affinity group is present in the flattenLineage, that needs to be handled specially
      IdentityHashSet<Tuple> flattenElements = affinityGroups.get(1);
     
      IdentityHashSet<Tuple> newAffinityGroup = new IdentityHashSet<Tuple>();
      if(currentOp instanceof LOEval) {
        EvalSpec spec = ((LOEval)currentOp).getSpec();
        if(spec instanceof FilterSpec) {
          //here we need to put a Tuple that doesn't pass the filter into the affinity groups.
          LogicalOperator input = currentOp.getOpTable().get(currentOp.getInputs().get(0)); //since its a filter, we know there is only 1 input.
          DataBag inputData = derivedData.get(input);
          for(Iterator<Tuple> it = inputData.iterator(); it.hasNext();) {
View Full Code Here

   
    static Collection<IdentityHashSet<Tuple>> GetEquivalenceClasses(LOEval op, Map<LogicalOperator, DataBag> derivedData) throws IOException {
       
        Collection<IdentityHashSet<Tuple>> equivClasses = new LinkedList<IdentityHashSet<Tuple>>();
       
        EvalSpec spec = op.getSpec();
       
        if (spec instanceof FilterSpec) {    // it's a FILTER
            FilterSpec fSpec = (FilterSpec) spec;

            // make two equivalence classes: one for tuples that pass the filter, and one for tuples that fail the filter
View Full Code Here

   
    private void setupMapPipe(Properties properties, Reporter reporter)
    throws IOException {
        SliceWrapper split = PigInputFormat.getActiveSplit();
        index = split.getIndex();
        EvalSpec evalSpec = split.getEvalSpec();
       
        EvalSpec groupSpec = split.getGroupbySpec();
        if (groupSpec == null) {

            String fileName = "map-" + getTaskId();
           
            SplitSpec splitSpec = (SplitSpec) ObjectSerializer.deserialize(job.get("pig.splitSpec", ""));
View Full Code Here

            }
        };
    }
   
    private void setupReducePipe(Properties properties) throws IOException {
        EvalSpec evalSpec = (EvalSpec)ObjectSerializer.deserialize(job.get("pig.reduceFunc", ""));
       
        if (evalSpec == null)
            evalSpec = new StarSpec();
        else
            evalSpec.instantiateFunc(pigContext);
       
        ArrayList<EvalSpec> groupSpecs = (ArrayList<EvalSpec>) ObjectSerializer.deserialize(job.get("pig.groupFuncs", ""));       
       
        isInner = new boolean[groupSpecs.size()];
        for (int i = 0; i < groupSpecs.size(); i++) {
            isInner[i] = groupSpecs.get(i).isInner();
           
        }

        SplitSpec splitSpec = (SplitSpec) ObjectSerializer.deserialize(job.get("pig.splitSpec", ""));       
        if (splitSpec != null){
            splitSpec.instantiateFunc(pigContext);
            oc = getSplitCollector(splitSpec);
        }
   
        evalPipe = evalSpec.setupPipe(properties,
                                      new ReduceDataOutputCollector());
       
        inputCount = ((ArrayList<FileSpec>)ObjectSerializer.deserialize(job.get("pig.inputs"))).size();

    }
View Full Code Here

    public void visitStore(LOStore s) {
        super.visitStore(s);
       
        if (parentEval) {
            EvalSpec spec = eval.getSpec();
            if (spec instanceof StreamSpec) {
                // Try and optimize if the store and stream output specs match
                StreamSpec streamSpec = (StreamSpec)spec;
                StreamingCommand command = streamSpec.getCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
View Full Code Here

    public void visitEval(LOEval e) {
        super.visitEval(e);

        if (parentLoad) {
            EvalSpec spec = e.getSpec();
            if (spec instanceof StreamSpec && !load.isSplittable()) {
                // Try and optimize if the load and stream input specs match
                // and input files are to be processed as-is
                StreamSpec streamSpec = (StreamSpec)spec;
                StreamingCommand command = streamSpec.getCommand();
View Full Code Here

            //mro.toMap.list.get(index).add(lo.spec);
           
            mro.mapParallelism = Math.max(mro.mapParallelism, lo.getRequestedParallelism());

        } else { // push into "reduce" phase
            EvalSpec spec = lo.getSpec();

            if (mro.toReduce == null && shouldCombine(spec, mro)) {
                // Push this spec into the combiner.  But we also need to
                // create a new spec with a changed expected projection to
                // push into the reducer.

                if (mro.toCombine != null) {
                    throw new AssertionError("Combiner already set.");
                }
                // mro.toCombine = spec;

                // Now, we need to adjust the expected projection for the
                // eval spec(s) we just pushed.  Also, this will change the
                // function to be the final instead of general instance.
                EvalSpec newSpec = spec.copy(pigContext);
                newSpec.visit(new ReduceAdjuster(pigContext));
                mro.addReduceSpec(newSpec);

                // Adjust the function name for the combine spec, to set it
                // to the initial function instead of the general
                // instance.  Make a copy of the eval spec rather than
                // adjusting the existing one, to prevent breaking the
                // logical plan in case another physical plan is generated
                // from it later.
                EvalSpec combineSpec = spec.copy(pigContext);
                combineSpec.visit(new CombineAdjuster());
                mro.toCombine = combineSpec;

            } else {
                mro.addReduceSpec(lo.getSpec()); // otherwise, don't use combiner
            }
View Full Code Here

        //find the quantiles in the reduce step
        ArrayList<EvalSpec> argsList = new ArrayList<EvalSpec>();
        argsList.add(new ConstSpec(Math.max(loSort.getRequestedParallelism()-1,1)));
       
        //sort the first column of the cogroup output and feed it to the quantiles function
        EvalSpec sortedSampleSpec = new ProjectSpec(1);
        EvalSpec starSpec = new StarSpec();
        starSpec.setComparatorName(loSort.getSortSpec().getComparatorName());
        sortedSampleSpec = sortedSampleSpec.addSpec(new SortDistinctSpec(false, starSpec));
        argsList.add(sortedSampleSpec);
       
        EvalSpec args = new GenerateSpec(argsList);

        EvalSpec reduceSpec = new FuncEvalSpec(pigContext, FindQuantiles.class.getName(), args);
        reduceSpec.setFlatten(true);
        quantileJob.addReduceSpec(new GenerateSpec(reduceSpec));
       
        //a temporary file to hold the quantile data
        String quantileFile = getTempFile(pigContext);
        quantileJob.outputFileSpec = new FileSpec(quantileFile, BinStorage.class.getName());
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.eval.EvalSpec$UserComparator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.