Examples of org.apache.pig.data.SortedDataBag

org.apache.pig.data.SortedDataBag
An ordered collection of Tuples (possibly) with multiples. Data is stored unsorted as it comes in, and only sorted when it is time to dump it to a file or when the first iterator is requested. Experementation found this to be the faster than storing it sorted to begin with. We allow a user defined comparator, but provide a default comparator in cases where the user doesn't specify one.

    private void accumulateData() throws ExecException {
  int size = inputs.size();
  its = new Iterator[size];
  data = new Tuple[size];
  for(int i = 0; i < size; i++) {
      DataBag bag = new SortedDataBag(new groupComparator());
      for(Result input = inputs.get(i).getNext(dummyTuple); input.returnStatus != POStatus.STATUS_EOP; input = inputs.get(i).getNext(dummyTuple)) {
          if(input.returnStatus == POStatus.STATUS_ERR) {
              throw new ExecException("Error accumulating output at local Cogroup operator");
          }
          if(input.returnStatus == POStatus.STATUS_NULL)
              continue;
          bag.add((Tuple) input.result);
      }
      
      its[i] = bag.iterator();
      data[i] = its[i].next();
  }


    }

View Full Code Here

    private void accumulateData() throws ExecException {
  int size = inputs.size();
  its = new Iterator[size];
  data = new Tuple[size];
  for(int i = 0; i < size; i++) {
      DataBag bag = new SortedDataBag(new groupComparator());
      for(Result input = inputs.get(i).getNext(dummyTuple); input.returnStatus != POStatus.STATUS_EOP; input = inputs.get(i).getNext(dummyTuple)) {
          if(input.returnStatus == POStatus.STATUS_ERR) {
              throw new ExecException("Error accumulating output at local Cogroup operator");
          }
          if(input.returnStatus == POStatus.STATUS_NULL)
              continue;
          bag.add((Tuple) input.result);
      }
      
      its[i] = bag.iterator();
      data[i] = its[i].next();
  }


    }

View Full Code Here

    private void accumulateData() throws ExecException {
  int size = inputs.size();
  its = new Iterator[size];
  data = new Tuple[size];
  for(int i = 0; i < size; i++) {
      DataBag bag = new SortedDataBag(new groupComparator());
      for(Result input = inputs.get(i).getNext(dummyTuple); input.returnStatus != POStatus.STATUS_EOP; input = inputs.get(i).getNext(dummyTuple)) {
          if(input.returnStatus == POStatus.STATUS_ERR) {
              throw new ExecException("Error accumulating output at local Cogroup operator");
          }
          if(input.returnStatus == POStatus.STATUS_NULL)
              continue;
          bag.add((Tuple) input.result);
      }
      
      its[i] = bag.iterator();
      data[i] = its[i].next();
  }


    }

View Full Code Here

    private void accumulateData() throws ExecException {
  int size = inputs.size();
  its = new Iterator[size];
  data = new Tuple[size];
  for(int i = 0; i < size; i++) {
      DataBag bag = new SortedDataBag(new groupComparator());
      for(Result input = inputs.get(i).getNext(dummyTuple); input.returnStatus != POStatus.STATUS_EOP; input = inputs.get(i).getNext(dummyTuple)) {
          if(input.returnStatus == POStatus.STATUS_ERR) {
              throw new ExecException("Error accumulating output at local Cogroup operator");
          }
          if(input.returnStatus == POStatus.STATUS_NULL)
              continue;
          bag.add((Tuple) input.result);
      }
      
      its[i] = bag.iterator();
      data[i] = its[i].next();
  }


    }

View Full Code Here

    private void accumulateData() throws ExecException {
  int size = inputs.size();
  its = new Iterator[size];
  data = new Tuple[size];
  for(int i = 0; i < size; i++) {
      DataBag bag = new SortedDataBag(new groupComparator());
      for(Result input = inputs.get(i).getNext(dummyTuple); input.returnStatus != POStatus.STATUS_EOP; input = inputs.get(i).getNext(dummyTuple)) {
          if(input.returnStatus == POStatus.STATUS_ERR) {
              throw new ExecException("Error accumulating output at local Cogroup operator");
          }
          if(input.returnStatus == POStatus.STATUS_NULL)
              continue;
          bag.add((Tuple) input.result);
      }
      
      its[i] = bag.iterator();
      data[i] = its[i].next();
  }


    }

View Full Code Here

    private void accumulateData() throws ExecException {
  int size = inputs.size();
  its = new Iterator[size];
  data = new Tuple[size];
  for(int i = 0; i < size; i++) {
      DataBag bag = new SortedDataBag(new groupComparator());
      for(Result input = inputs.get(i).getNext(dummyTuple); input.returnStatus != POStatus.STATUS_EOP; input = inputs.get(i).getNext(dummyTuple)) {
          if(input.returnStatus == POStatus.STATUS_ERR) {
              throw new ExecException("Error accumulating output at local Cogroup operator");
          }
          if(input.returnStatus == POStatus.STATUS_NULL)
              continue;
          bag.add((Tuple) input.result);
      }
      
      its[i] = bag.iterator();
      data[i] = its[i].next();
  }


    }

View Full Code Here

TOP

Related Classes of org.apache.pig.data.SortedDataBag

org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POCogroup

org.apache.pig.pen.physicalOperators.POCogroup

java.io.File

java.io.DataOutputStream

java.io.FileInputStream

java.io.BufferedInputStream

java.io.DataInputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.