Examples of FuncSpec

com.sap.hadoop.windowing.query.FuncSpec
org.apache.pig.FuncSpec
Class to represent a UDF specification. Encapsulates the class name and the arguments to the constructor.

Examples of org.apache.pig.FuncSpec


        @Override
        public void transform(OperatorPlan matched) throws FrontendException {
            for (LogicalExpressionPlan expPlan : expPlansToInsertIdentity) {
                LogicalExpression oldRoot = (LogicalExpression)expPlan.getSources().get(0);
                UserFuncExpression userFuncExpression = new UserFuncExpression(expPlan, new FuncSpec(IdentityColumn.class.getName()));
                expPlan.connect(userFuncExpression, oldRoot);
            }
            expPlansToInsertIdentity.clear();
        }

View Full Code Here

Examples of org.apache.pig.FuncSpec

     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
        funcList.add(new FuncSpec(DoubleMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
        funcList.add(new FuncSpec(FloatMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
        funcList.add(new FuncSpec(IntMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
        funcList.add(new FuncSpec(LongMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
        funcList.add(new FuncSpec(StringMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.CHARARRAY)));
        return funcList;
    }

View Full Code Here

Examples of org.apache.pig.FuncSpec

     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
        funcList.add(new FuncSpec(DoubleMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
        funcList.add(new FuncSpec(FloatMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
        funcList.add(new FuncSpec(IntMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
        funcList.add(new FuncSpec(LongMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
        funcList.add(new FuncSpec(StringMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.CHARARRAY)));
        return funcList;
    }

View Full Code Here

Examples of org.apache.pig.FuncSpec

                String scope = leaf.getOperatorKey().getScope();
                POStore str = new POStore(new OperatorKey(scope,
                    NodeIdGenerator.getGenerator().getNextNodeId(scope)));
                spec = new FileSpec(FileLocalizer.getTemporaryPath(
                    pigContext).toString(),
                    new FuncSpec(Utils.getTmpFileCompressorName(pigContext)));
                str.setSFile(spec);
                plan.addAsLeaf(str);
            } else{
                spec = ((POStore)leaf).getSFile();
            }

View Full Code Here

Examples of org.apache.pig.FuncSpec

     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
        funcList.add(new FuncSpec(DoubleSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
        funcList.add(new FuncSpec(FloatSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
        funcList.add(new FuncSpec(IntSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
        funcList.add(new FuncSpec(LongSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
        return funcList;
    }

View Full Code Here

Examples of org.apache.pig.FuncSpec

        LogicalExpression copy = new CastExpression(
                lgExpPlan,
                this.getExpression().deepCopy(lgExpPlan),
                this.getFieldSchema().deepCopy());
        try {
            FuncSpec origFuncSpec = this.getFuncSpec();
            if (origFuncSpec != null ) {
                ((CastExpression)copy).setFuncSpec(origFuncSpec.clone());
            }
        } catch(CloneNotSupportedException e) {
            e.printStackTrace(); 
        }
        return copy;

View Full Code Here

Examples of org.apache.pig.FuncSpec

        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        Schema s = new Schema();
        s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
        s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
        s.add(new Schema.FieldSchema(null, DataType.INTEGER));
        funcList.add(new FuncSpec(this.getClass().getName(), s));
        return funcList;
    }

View Full Code Here

Examples of org.apache.pig.FuncSpec

          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.BYTEARRAY));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.BYTEARRAY));
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, lp, null);
          A.setAlias("A");
          lp.add(A);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          LOLoad B = new LOLoad(new FileSpec("morebla", new FuncSpec("PigStorage", "\t")), bschema, lp, null);
          B.setAlias("B");
          lp.add(B);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          MultiMap<Integer, LogicalExpressionPlan> mm = 
                new MultiMap<Integer, LogicalExpressionPlan>();
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
          new ProjectExpression(aprojplan, 0, 0, C);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          new ProjectExpression(bprojplan, 1, 0, C);
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          
          C.setAlias("C");
          lp.add(C);
          lp.connect(A, C);
          lp.connect(B, C);
        
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          LOFilter D = new LOFilter(lp, filterPlan);
          ProjectExpression fx = new ProjectExpression(filterPlan, 0, 0, D);
          ConstantExpression fc0 = new ConstantExpression(filterPlan, new Integer(0), new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          EqualExpression eq1 = new EqualExpression(filterPlan, fx, fc0);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, 0, 0, D);
          ProjectExpression fa = new ProjectExpression(filterPlan, 0, 2, D);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          AndExpression and1 = new AndExpression(filterPlan, eq1, eq2);
          ProjectExpression fb = new ProjectExpression(filterPlan, 0, 3, D);
          ConstantExpression fc1 = new ConstantExpression(filterPlan, new Integer(1),new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          EqualExpression eq3 = new EqualExpression(filterPlan, fb, fc1);
          AndExpression and2 = new AndExpression(filterPlan, and1, eq3);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, 0, 3, D);
          ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, and2, eq4);
          
          D.setAlias("D");
          // Connect D to B, since the transform has happened.
          lp.add(D);
          lp.connect(C, D);
        }
        
        System.out.println(lp);
        LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(lp, 500, null);
        optimizer.optimize();
        
        LogicalPlan expected = new LogicalPlan();
        {
            // A = load
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.BYTEARRAY));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.BYTEARRAY));
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, expected, null);
          expected.add(A);
          
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, 0, 0, DA);
          fx.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, new Integer(0), new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          new EqualExpression(DAfilterPlan, fx, fc0);
          
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          LOLoad B = new LOLoad(new FileSpec("morebla", new FuncSpec("PigStorage", "\t")), bschema, expected, null);
          expected.add(B);
          
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
            LOFilter DB = new LOFilter(expected, DBfilterPlan);

View Full Code Here

Examples of org.apache.pig.FuncSpec

            }
            
        }
        
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            try {
                pc.connect();

View Full Code Here

Examples of org.apache.pig.FuncSpec

                String scope = leaf.getOperatorKey().getScope();
                POStore str = new POStore(new OperatorKey(scope,
                    NodeIdGenerator.getGenerator().getNextNodeId(scope)));
                spec = new FileSpec(FileLocalizer.getTemporaryPath(
                    pigContext).toString(),
                    new FuncSpec(Utils.getTmpFileCompressorName(pigContext)));
                str.setSFile(spec);
                plan.addAsLeaf(str);
            } else{
                spec = ((POStore)leaf).getSFile();
            }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.