Package org.apache.pig.data

Examples of org.apache.pig.data.Datum


public class IsEmpty extends FilterFunc {

    @Override
    public boolean exec(Tuple input) throws IOException {
        Datum values = input.getField(0);
        if (values instanceof DataBag) {
            return ((DataBag)values).size() == 0;
        } else if (values instanceof DataMap) {
            return ((DataMap)values).cardinality() == 0;
        } else {
            StringBuilder sb = new StringBuilder();
            sb.append("Cannot test a ");
            sb.append(values.getClass().getSimpleName());
            sb.append(" for emptiness.");
            throw new IOException(sb.toString());
        }
    }
View Full Code Here


           
            Tuple inputT = new Tuple(outputConstraint.arity());
            ExampleTuple inputTuple = new ExampleTuple();
            inputTuple.copyFrom(inputT);
            for (int i = 0; i < inputTuple.arity(); i++) {
                Datum d = outputConstraint.getField(i);
                if (d == null) d = exampleTuple.getField(i);
                inputTuple.setField(i, d);
            }
            if(inputTuple.equals(exampleTuple)) {
              //System.out.println("Real tuple being copied!!");
View Full Code Here

        if (ableToHandle) {
            // first, go through output constraints and create new groups
            for (Iterator<Tuple> it = outputConstraints.iterator(); it.hasNext(); ) {
                Tuple outputConstraint = it.next();
                Datum groupLabel = outputConstraint.getField(0);
               
                for (int input = 0; input < numInputs; input++) {
                    //int numInputFields = inputs.get(input).outputSchema().numFields();
                  int numInputFields = op.getOpTable().get(inputs.get(input)).outputSchema().numFields();
//                    List<Integer> groupCols = groupSpecs.get(input).getCols();
                    List<Integer> groupCols = groupSpecs.get(input);
                   
                    for (int i = 0; i < minGroupSize; i++) {
                        Tuple inputConstraint = GenerateGroupByInput(groupLabel, groupCols, numInputFields);
                        if (inputConstraint != null) inputConstraints.get(input).add(inputConstraint);
                    }
                }
            }
       
            // then, go through all organic data groups and add input constraints to make each group big enough
            DataBag outputData = derivedData.get(op);
           
            for (Iterator<Tuple> it = outputData.iterator(); it.hasNext(); ) {
              Tuple groupTup = it.next();
              Datum groupLabel = groupTup.getField(0);

              for (int input = 0; input < numInputs; input++) {
                int numInputFields = op.getOpTable().get(inputs.get(input)).outputSchema().numFields();
//              List<Integer> groupCols = groupSpecs.get(input).getCols();
                List<Integer> groupCols = groupSpecs.get(input);                   
View Full Code Here

                if (!(groupLabel instanceof Tuple)) throw new RuntimeException("Unexpected group label type.");
                Tuple groupLabelTuple = (Tuple) groupLabel;
                       
                for (int outCol = 0; outCol < groupCols.size(); outCol++) {
                    int inCol = groupCols.get(outCol);
                    Datum outVal = groupLabelTuple.getField(outCol);
                    inputConstraint.setField(inCol, outVal);
                }
            }
        }
        return inputConstraint;
View Full Code Here

        Tuple inputConstraint = new ExampleTuple();
        inputConstraint.copyFrom(inputConst);

        for (int outCol = 0; outCol < outputConstraint.arity(); outCol++) {
            int inCol = cols.get(outCol);
            Datum outVal = outputConstraint.getField(outCol);
            Datum inVal = inputConstraint.getField(inCol);
           
            if (inVal == null) {
                inputConstraint.setField(inCol, outVal);
            } else {
                if (outVal != null) {
View Full Code Here

   
    static void GenerateMatchingTupleHelper(Tuple t, CompCond pred, boolean invert) throws IOException {
       
        // decide if the eval functions on the two sides of the predicate are simple enough for us to deal with
        boolean leftIsConst = false, rightIsConst = false;
        Datum leftConst = null, rightConst = null;
        int leftCol = -1, rightCol = -1;
        if (pred.left instanceof ConstSpec) {
            leftIsConst = true;
            leftConst = ((ConstSpec) pred.left).atom;
        } else {
            if (!(pred.left instanceof ProjectSpec && ((ProjectSpec) pred.left).numCols() == 1)) return// too hard, give up
            leftCol = ((ProjectSpec) pred.left).getCol(0);
            Datum d = t.getField(leftCol);
            if (d != null) {
                leftIsConst = true;
                leftConst = d;
            }
        }
        if (pred.right instanceof ConstSpec) {
            rightIsConst = true;
            rightConst = ((ConstSpec) pred.right).atom;
        } else {
            if (!(pred.right instanceof ProjectSpec && ((ProjectSpec) pred.right).numCols() == 1)) return// too hard, give up
            rightCol = ((ProjectSpec) pred.right).getCol(0);
            Datum d = t.getField(rightCol);
            if (d != null) {
                rightIsConst = true;
                rightConst = d;
            }
        }
View Full Code Here

       
        while (true) { // loop until we find a tuple we're allowed to output (or we hit the end)

            // find the smallest group among all inputs (this is the group we should make a tuple
            // out of)
            Datum smallestGroup = null;
            for (int i = 0; i < inputs.length; i++) {
                if (sortedInputs[i].size() > 0) {
                    Datum g = (sortedInputs[i].get(0))[0];
                    if (smallestGroup == null || g.compareTo(smallestGroup)<0)
                        smallestGroup = g;
                }
            }

            if (smallestGroup == null)
                return null; // we hit the end of the groups, so we're done

            // find all tuples in each input pertaining to the group of interest, and combine the
            // data into a single tuple
           
            Tuple output;
            ExampleTuple tOut = new ExampleTuple();
            if (outputType == LogicalOperator.AMENDABLE) output = new AmendableTuple(1 + inputs.length, smallestGroup);
            else output = new Tuple(1 + inputs.length);

            // set first field to the group tuple
            output.setField(0, smallestGroup);
            tOut.copyFrom(output);
            if (lineageTracer != null) lineageTracer.insert(tOut);

            boolean done = true;
            for (int i = 0; i < inputs.length; i++) {
                DataBag b = BagFactory.getInstance().newDefaultBag();

                while (sortedInputs[i].size() > 0) {
                    Datum g = sortedInputs[i].get(0)[0];

                    Tuple t = (Tuple) sortedInputs[i].get(0)[1];

                    if (g.compareTo(smallestGroup) < 0) {
                        sortedInputs[i].remove(0); // discard this tuple
                    } else if (g.equals(smallestGroup)) {
                        b.add(t);
                        //if (lineageTracer != null) lineageTracer.union(t, output);   // update lineage
                        if (lineageTracer != null) {
                          if(((ExampleTuple)t).isSynthetic()) tOut.makeSynthetic();
                          lineageTracer.union(t, tOut);   // update lineage
View Full Code Here

            output.setValue(sum(input));
        }
    }

    static protected long count(Tuple input) throws IOException {
        Datum values = input.getField(0);       
        if (values instanceof DataBag)
            return ((DataBag)values).size();
        else if (values instanceof DataMap)
            return ((DataMap)values).cardinality();
        else
            throw new IOException("Cannot count a " + values.getClass().getSimpleName());
    }
View Full Code Here

    static String ShortenField(Tuple t) {
      StringBuffer str = new StringBuffer();
      int noFields = t.arity();
      str.append("(");
      if(noFields > 3) {
        Datum d = t.getField(0);
     
      str.append(ShortenField(d) + ", ..., ");
      d = t.getField(noFields - 1);
     
      str.append(ShortenField(d));
       
      } else {
        for(int i = 0; i < noFields; ++i) {
          Datum d;
        d = t.getField(i);
       
        if(i != (noFields - 1)) {
          str.append(ShortenField(d) + ", ");
        } else {
View Full Code Here

  public void exec(Tuple input, DataAtom output) throws IOException {
    output.setValue(toRadian(input));
  }
 
  protected double toRadian(Tuple input) throws IOException{
    Datum temp = input.getField(0);
    double retVal;
    if(!(temp instanceof DataAtom)){
      throw new IOException("invalid input format. ");
    }
    else{
View Full Code Here

TOP

Related Classes of org.apache.pig.data.Datum

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.