Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct
Find the distinct set of tuples in a bag. This is a blocking operator. All the input is put in the hashset implemented in DistinctDataBag which also provides the other DataBag interfaces.

    }


    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelism());
        physOp.setCustomPartitioner(loDistinct.getCustomPartitioner());
        physOp.addOriginalLocation(loDistinct.getAlias(), loDistinct.getLocation());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                numDistinctChanged++;
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

    }
    
    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelisam());
        physOp.setAlias(loDistinct.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                numDistinctChanged++;
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

    public void confirmDistinct() throws ExecException {
       
      PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
      List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
      inputs.add(read);
      PODistinct distinct = new PODistinct(new OperatorKey("", r.nextLong()),
              -1, inputs);
      Map<Tuple, Integer> output = new HashMap<Tuple, Integer>();
      Tuple t = null;
      Result res = distinct.getNext(t);
      t = (Tuple) res.result;
      while (res.returnStatus != POStatus.STATUS_EOP) {
          if (output.containsKey(t)) {
              int i = output.get(t);
              output.put(t, ++i);
          } else {
              output.put(t, 1);
          }
          res = distinct.getNext(t);
          t = (Tuple) res.result;
      }
      for (Map.Entry<Tuple, Integer> e : output.entrySet()) {
          int i = e.getValue();
          // System.out.println(e.getKey());

View Full Code Here

    }
    
    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelisam());
        physOp.setAlias(loDistinct.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

    public void testDistinct1() throws Exception {
        PhysicalPlan php = new PhysicalPlan();
        PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
        php.merge(ldFil1);


        PODistinct op = new PODistinct(new OperatorKey("", r.nextLong()),
                -1, null);


        php.addAsLeaf(op);


        PhysicalPlan grpChain1 = GenPhyOp.grpChain();
        php.merge(grpChain1);
        php.connect(op,grpChain1.getRoots().get(0));


        PODistinct op1 = new PODistinct(new OperatorKey("", r.nextLong()),
                -1, null);


        php.addAsLeaf(op1);
        POStore st = GenPhyOp.topStoreOp();
        php.addAsLeaf(st);

View Full Code Here

        PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain();
        php.merge(grpChain1);


        List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
        PhysicalPlan inplan = new PhysicalPlan();
        PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()),
                -1, null);
        inplan.addAsLeaf(op1);
        inputs.add(inplan);
        List<Boolean> toFlattens = new ArrayList<Boolean>();
        toFlattens.add(false);

View Full Code Here

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                numDistinctChanged++;
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer

org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result

org.apache.pig.backend.hadoop.executionengine.util.SecondaryKeyOptimizerUtil

org.apache.pig.data.InternalDistinctBag

org.apache.pig.impl.plan.OperatorKey

org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor

org.apache.pig.test.TestMRCompiler

org.apache.pig.test.TestPODistinct

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.