Examples of FuncSpec


Examples of org.apache.pig.FuncSpec

  PigServer server = new PigServer(ExecType.LOCAL);
  server.setBatchOn();
  server.registerFunction(
    "org.apache.pig.piggybank.storage.HiveColumnarLoader",
    new FuncSpec(funcSpecString));

  server.registerQuery("a = LOAD '"
    + datePartitionedDir.getAbsolutePath() + "' using "
    + funcSpecString + ";");
  Iterator<Tuple> result = server.openIterator("a");
View Full Code Here

Examples of org.apache.pig.FuncSpec

         * the candidate with the lowest score is chosen.
         */
       
       
       
        FuncSpec matchingSpec = null;
        boolean notExactMatch = false;
        if(funcSpecs!=null && funcSpecs.size()!=0){
            //Some function mappings found. Trying to see
            //if one of them fits the input schema
            if((matchingSpec = exactMatch(funcSpecs, s, func))==null){
                //Oops, no exact match found. Trying to see if we
                //have mappings that we can fit using casts.
                notExactMatch = true;
                if(byteArrayFound(s)){
                    // try "exact" matching all other fields except the byte array
                    // fields and if they all exact match and we have only one candidate
                    // for the byte array cast then that's the matching one!
                    if((matchingSpec = exactMatchWithByteArrays(funcSpecs, s, func))==null){
                        // "exact" match with byte arrays did not work - try best fit match
                        if((matchingSpec = bestFitMatchWithByteArrays(funcSpecs, s, func)) == null) {
                            int errCode = 1045;
                            String msg = "Could not infer the matching function for "
                                + func.getFuncSpec()
                                + " as multiple or none of them fit. Please use an explicit cast.";
                            msgCollector.collect(msg, MessageType.Error);
                            throw new TypeCheckerException(msg, errCode, PigException.INPUT);
                        }
                    }
                } else if ((matchingSpec = bestFitMatch(funcSpecs, s)) == null) {
                    // Either no byte arrays found or there are byte arrays
                    // but only one mapping exists.
                    // However, we could not find a match as there were either
                    // none fitting the input schema or it was ambiguous.
                    // Throw exception that we can't infer a fit.
                    int errCode = 1045;
                    String msg = "Could not infer the matching function for "
                            + func.getFuncSpec()
                            + " as multiple or none of them fit. Please use an explicit cast.";
                    msgCollector.collect(msg, MessageType.Error);
                    throw new TypeCheckerException(msg, errCode, PigException.INPUT);
                }
            }
        }
        if(matchingSpec!=null){
            //Voila! We have a fitting match. Lets insert casts and make
            //it work.
            // notify the user about the match we picked if it was not
            // an exact match
            if(notExactMatch) {
                String msg = "Function " + func.getFuncSpec().getClassName() + "()" +
                             " will be called with following argument types: " +
                             matchingSpec.getInputArgsSchema() + ". If you want to use " +
                             "different input argument types, please use explicit casts.";
                msgCollector.collect(msg, MessageType.Warning, PigWarning.USING_OVERLOADED_FUNCTION);
            }
            matchingSpec.setCtorArgs(func.getFuncSpec().getCtorArgs());
            func.setFuncSpec(matchingSpec);
            insertCastsForUDF(func, s, matchingSpec.getInputArgsSchema());
           
        }
           
        //Regenerate schema as there might be new additions
        try {
View Full Code Here

Examples of org.apache.pig.FuncSpec

     * @return the funcSpec that supports the schema that is best suited
     *          to s. The best suited schema is one that has the
     *          lowest score as returned by fitPossible().
     */
    private FuncSpec bestFitMatch(List<FuncSpec> funcSpecs, Schema s) {
        FuncSpec matchingSpec = null;
        long score = INF;
        long prevBestScore = Long.MAX_VALUE;
        long bestScore = Long.MAX_VALUE;
        for (Iterator<FuncSpec> iterator = funcSpecs.iterator(); iterator.hasNext();) {
            FuncSpec fs = iterator.next();
            score = fitPossible(s,fs.getInputArgsSchema());
            if(score!=INF && score<=bestScore){
                matchingSpec = fs;
                prevBestScore = bestScore;
                bestScore = score;
            }
View Full Code Here

Examples of org.apache.pig.FuncSpec

    private FuncSpec bestFitMatchWithByteArrays(List<FuncSpec> funcSpecs,
            Schema s, LOUserFunc func) throws VisitorException {
    List<Pair<Long, FuncSpec>> scoreFuncSpecList = new ArrayList<Pair<Long,FuncSpec>>();
        for (Iterator<FuncSpec> iterator = funcSpecs.iterator(); iterator
                .hasNext();) {
            FuncSpec fs = iterator.next();
            long score = fitPossible(s, fs.getInputArgsSchema());
            if (score != INF) {
                scoreFuncSpecList.add(new Pair<Long, FuncSpec>(score, fs));
            }
        }

        // if no candidates found, return null
        if(scoreFuncSpecList.size() == 0)
            return null;
       
        if(scoreFuncSpecList.size() > 1) {
            // sort the candidates based on score
            Collections.sort(scoreFuncSpecList, new ScoreFuncSpecListComparator());
           
            // if there are two (or more) candidates with the same *lowest* score
            // we cannot choose one of them - notify the user
            if (scoreFuncSpecList.get(0).first == scoreFuncSpecList.get(1).first) {
                int errCode = 1046;
                String msg = "Multiple matching functions for "
                        + func.getFuncSpec() + " with input schemas: " + "("
                        + scoreFuncSpecList.get(0).second.getInputArgsSchema() + ", "
                        + scoreFuncSpecList.get(1).second.getInputArgsSchema() + "). Please use an explicit cast.";
                msgCollector.collect(msg, MessageType.Error);
                throw new TypeCheckerException(msg, errCode, PigException.INPUT);
            }
       
            // now consider the bytearray fields
            List<Integer> byteArrayPositions = getByteArrayPositions(s);
            // make sure there is only one type to "cast to" for the byte array
            // positions among the candidate funcSpecs
            Map<Integer, Pair<FuncSpec, Byte>> castToMap = new HashMap<Integer, Pair<FuncSpec, Byte>>();
            for (Iterator<Pair<Long, FuncSpec>> it = scoreFuncSpecList.iterator(); it.hasNext();) {
                FuncSpec funcSpec = it.next().second;
                Schema sch = funcSpec.getInputArgsSchema();
                for (Iterator<Integer> iter = byteArrayPositions.iterator(); iter
                        .hasNext();) {
                    Integer i = iter.next();
                    try {
                        if (!castToMap.containsKey(i)) {
                            // first candidate
                            castToMap.put(i, new Pair<FuncSpec, Byte>(funcSpec, sch
                                    .getField(i).type));
                        } else {
                            // make sure the existing type from an earlier candidate
                            // matches
                            Pair<FuncSpec, Byte> existingPair = castToMap.get(i);
                            if (sch.getField(i).type != existingPair.second) {
                                int errCode = 1046;
                                String msg = "Multiple matching functions for "
                                        + func.getFuncSpec() + " with input schema: "
                                        + "(" + existingPair.first.getInputArgsSchema()
                                        + ", " + funcSpec.getInputArgsSchema()
                                        + "). Please use an explicit cast.";
                                msgCollector.collect(msg, MessageType.Error);
                                throw new TypeCheckerException(msg, errCode, PigException.INPUT);
                            }
                        }
View Full Code Here

Examples of org.apache.pig.FuncSpec

     * @throws VisitorException
     */
    private FuncSpec exactMatchHelper(List<FuncSpec> funcSpecs, Schema s, LOUserFunc func, boolean ignoreByteArrays) throws VisitorException {
        List<FuncSpec> matchingSpecs = new ArrayList<FuncSpec>();
        for (Iterator<FuncSpec> iterator = funcSpecs.iterator(); iterator.hasNext();) {
            FuncSpec fs = iterator.next();
            if (schemaEqualsForMatching(s, fs.getInputArgsSchema(), ignoreByteArrays)) {
                matchingSpecs.add(fs);
            }
        }
        if(matchingSpecs.size() == 0)
            return null;
View Full Code Here

Examples of org.apache.pig.FuncSpec

        ) {
            try {
              Map<String, LogicalOperator> canonicalMap = cast.getFieldSchema().getCanonicalMap();
              // two variables to ensure that only one load func is mapped to
              // the cast operator
              FuncSpec prevLoadFuncSpec = null;
              boolean prevLoadFuncSet = false;
              for( Map.Entry<String, LogicalOperator> entry : canonicalMap.entrySet() ) {
                    FuncSpec loadFuncSpec = getLoadFuncSpec( entry.getValue(), entry.getKey() );
                    if(!prevLoadFuncSet){
                        prevLoadFuncSet = true;
                        prevLoadFuncSpec = loadFuncSpec;
                    }
                    if(loadFuncSpec == null ){
                        if(prevLoadFuncSpec != null){
                            //BUG
                            String msg = "Bug: A null and a non-null load function " +
                            " mapped to cast through lineage";
                            throw new VisitorException(msg, 2258, PigException.BUG);
                        }else{
                            continue;
                        }
                    }
                    if(!loadFuncSpec.equals(prevLoadFuncSpec)){
                        //BUG
                        String msg = "Bug:Two different load functions mapped to " +
                        "an LOCast op";
                        throw new VisitorException(msg, 2258, PigException.BUG);
                    }
View Full Code Here

Examples of org.apache.pig.FuncSpec

            if(op instanceof LOLoad) {
                return ((LOLoad)op).getInputFile().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec();
                FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
                return streamLoaderSpec;
            }
           
            Schema s = op.getSchema();
            if(null != s) {
              FieldSchema fieldSchema = s.findFieldSchema( parentCanonicalName );
                getLoadFuncSpec( fieldSchema, loadFuncSpecMap );
            } else {
                LogicalPlan lp = op.getPlan();
                for(LogicalOperator pred: lp.getPredecessors(op)) {
                    FuncSpec lfSpec = getLoadFuncSpec(pred, parentCanonicalName);
                    if(null != lfSpec) loadFuncSpecMap.put(lfSpec.getClassName(), lfSpec);
                }
            }
        }
        if(loadFuncSpecMap.keySet().size() == 0) {
            return null;
View Full Code Here

Examples of org.apache.pig.FuncSpec

        return;
     
        Map<String, LogicalOperator> canonicalMap = fieldSchema.getCanonicalMap();
        if( canonicalMap.size() > 0 ) {
          for(Map.Entry<String, LogicalOperator> entry : canonicalMap.entrySet() ) {
                FuncSpec lfSpec = getLoadFuncSpec( entry.getValue(), entry.getKey() );
                if( null != lfSpec )
                  loadFuncSpecMap.put( lfSpec.getClassName(), lfSpec );
          }
        } else {
          MultiMap<LogicalOperator, String> reverseCanonicalMap = fieldSchema.getReverseCanonicalMap();
          for( LogicalOperator lop : reverseCanonicalMap.keySet() ) {
            FuncSpec lfSpec = getLoadFuncSpec( lop, null );
            if( null != lfSpec )
              loadFuncSpecMap.put( lfSpec.getClassName(), lfSpec );
          }
        }
    }
View Full Code Here

Examples of org.apache.pig.FuncSpec

        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:chararray,y:int);");
       
        DataBag dbfrj = BagFactory.getInstance().newDefaultBag(), dbshj = BagFactory.getInstance().newDefaultBag();
        {
            String fSpec = FRJoin.class.getName()+ "('" + INPUT_FILE + "')";
            pigServer.registerFunction("FRJ", new FuncSpec(fSpec));
            pigServer.registerQuery("C = foreach A generate *, flatten(FRJ(*));");
            Iterator<Tuple> iter = pigServer.openIterator("C");
           
            while(iter.hasNext()) {
                dbfrj.add(iter.next());
View Full Code Here

Examples of org.apache.pig.FuncSpec

     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY))));

        return funcList;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.