Examples of org.apache.pig.LoadFunc$RequiredField

org.apache.pig.LoadFunc
This interface is used to implement functions to parse records from a dataset. This also includes functions to cast raw byte data into various datatypes. These are external functions because we want loaders, whenever possible, to delay casting of datatypes until the last possible moment (i.e. don't do it on load). This means we need to expose the functionality so that other sections of the code can call back to the loader to do the cast.

                // Instantiate both LoadFunc objects to compare them for 
                // equality
                StoreFunc streamStorer = 
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
                LoadFunc inputLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(
                                             loadFileSpec.getFuncSpec());
                LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                        "inputLoader:" + inputLoader);
                // Check if the streaming command's inputSpec also implements 
                // LoadFunc and if it does, are they of the same _reversible_

View Full Code Here

            HandleSpec streamOutputSpec = command.getOutputSpec(); 
            
            FileSpec storeFileSpec = store.getOutputFile();
            
            // Instantiate both to compare them for equality
            LoadFunc streamLoader = 
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
            
            StoreFunc outputStorer = (StoreFunc)PigContext.instantiateFuncFromSpec(
                                         storeFileSpec.getFuncSpec());
            
            // Check if the streaming command's outputSpec also implements 
            // StoreFunc and if it does, are they of the same _reversible_ 
            // type?
            boolean sameType = false;
            try {
                // Check if the streamLoader is _reversible_ as 
                // the inputLoader ...
                if (streamLoader instanceof StoreFunc) {
                    // Cast to check if they are of the same type...
                    streamLoader.getClass().cast(outputStorer);
                    
                    // Now check if they both are reversible...
                    if (streamLoader instanceof ReversibleLoadStoreFunc &&
                        outputStorer instanceof ReversibleLoadStoreFunc) {
                        sameType = true;
                    }
                }
            } catch (ClassCastException cce) {
                sameType = false;
            }
            // Check if both LoadFunc objects belong to the same type and
            // are equivalent
            if (sameType && streamLoader.equals(outputStorer)) {
                // Since they both are the same, we can flip them 
                // for BinaryStorage
                mOptimizeStore = true;                    
            }
        }

View Full Code Here

            
                ld.setPc(pc);
                Tuple dummyTuple = null;
                for(Result res=ld.getNext(dummyTuple);res.returnStatus!=POStatus.STATUS_EOP;res=ld.getNext(dummyTuple)){
                    Tuple tup = (Tuple) res.result;
                    LoadFunc lf = ((LoadFunc)pc.instantiateFuncFromSpec(ld.getLFile().getFuncSpec()));
                    String key = lf.bytesToCharArray(((DataByteArray)tup.get(keyField)).get());
                    Tuple csttup = TupleFactory.getInstance().newTuple(2);
                    csttup.set(0, key);
                    csttup.set(1, lf.bytesToInteger(((DataByteArray)tup.get(1)).get()));
                    DataBag vals = null;
                    if(replTbl.containsKey(key)){
                        vals = replTbl.get(key);
                    }
                    else{

View Full Code Here

    public List<OperatorKey> getTargetOps() {
        return targetOps;
    }


    public LoadFunc getLoadFunction() {
        LoadFunc loader = null;
        if (this.parser == null) {
            loader = new PigStorage();
        } else {
            try {
                loader = (LoadFunc) PigContext

View Full Code Here

        if (!(r instanceof POLoad)) { // Huh?
            log.debug("Predecessor's map plan root is not a load.");
            return;
        }
        POLoad predLoad = (POLoad)r;
        LoadFunc lf = (LoadFunc)PigContext.instantiateFuncFromSpec(predLoad.getLFile().getFuncSpec());
        if (!(lf instanceof SamplableLoader)) {
            log.debug("Predecessor's loader does not implement SamplableLoader");
            return;
        }

View Full Code Here

            //Process the POLoads
            List<POLoad> lds = PlanHelper.getPhysicalOperators(mro.mapPlan, POLoad.class);


            if(lds!=null && lds.size()>0){
                for (POLoad ld : lds) {
                    LoadFunc lf = ld.getLoadFunc();
                    lf.setLocation(ld.getLFile().getFileName(), nwJob);


                    //Store the inp filespecs
                    inp.add(ld.getLFile());
                }
            }

View Full Code Here

    }


    String buildLoadOp(SourceLocation loc, String alias, String filename, FuncSpec funcSpec, LogicalSchema schema)
    throws ParserValidationException {
        String absolutePath;
        LoadFunc loFunc;
        try {
            // Load LoadFunc class from default properties if funcSpec is null. Fallback on PigStorage if LoadFunc is not specified in properties.
            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, PigStorage.class.getName())) : funcSpec;
            loFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(funcSpec);
            String fileNameKey = QueryParserUtils.constructFileNameSignature(filename, funcSpec) + "_" + (loadIndex++);
            absolutePath = fileNameMap.get(fileNameKey);
            if (absolutePath == null) {
                absolutePath = loFunc.relativeToAbsolutePath( filename, QueryParserUtils.getCurrentDir( pigContext ) );


                if (absolutePath!=null) {
                    QueryParserUtils.setHdfsServers( absolutePath, pigContext );
                }
                fileNameMap.put( fileNameKey, absolutePath );

View Full Code Here

    }


    String buildLoadOp(SourceLocation loc, String alias, String filename, FuncSpec funcSpec, LogicalSchema schema)
    throws ParserValidationException {
        String absolutePath;
        LoadFunc loFunc;
        try {
            // Load LoadFunc class from default properties if funcSpec is null. Fallback on PigStorage if LoadFunc is not specified in properties.
            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, PigStorage.class.getName())) : funcSpec;
            loFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(funcSpec);
            String fileNameKey = QueryParserUtils.constructFileNameSignature(filename, funcSpec) + "_" + (loadIndex++);
            absolutePath = fileNameMap.get(fileNameKey);
            if (absolutePath == null) {
                absolutePath = loFunc.relativeToAbsolutePath( filename, QueryParserUtils.getCurrentDir( pigContext ) );


                if (absolutePath!=null) {
                    QueryParserUtils.setHdfsServers( absolutePath, pigContext );
                }
                fileNameMap.put( fileNameKey, absolutePath );

View Full Code Here

        int n = pigSplit.getTotalSplits();
        context.getConfiguration().setInt("pig.mapsplits.count", n);
        Configuration conf = context.getConfiguration();
        PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer
                .deserialize(conf.get("udf.import.list")));
        LoadFunc loadFunc = getLoadFunc(pigSplit.getInputIndex(), conf);
        // Pass loader signature to LoadFunc and to InputFormat through
        // the conf
        passLoadSignature(loadFunc, pigSplit.getInputIndex(), conf);


        // merge entries from split specific conf into the conf we got
        PigInputFormat.mergeSplitSpecificConf(loadFunc, pigSplit, conf);


        // for backward compatibility
        PigInputFormat.sJob = conf;


        InputFormat inputFormat = loadFunc.getInputFormat();


        List<Long> inpLimitLists =
                (ArrayList<Long>)ObjectSerializer.deserialize(
                        conf.get("pig.inpLimits"));

View Full Code Here

                // input location into the configuration (for example,
                // FileInputFormat stores this in mapred.input.dir in the conf),
                // then for different inputs, the loader's don't end up
                // over-writing the same conf.
                FuncSpec loadFuncSpec = inputs.get(i).getFuncSpec();
                LoadFunc loadFunc = (LoadFunc) PigContext.instantiateFuncFromSpec(
                        loadFuncSpec);
                boolean combinable = !(loadFunc instanceof MergeJoinIndexer
                        || loadFunc instanceof IndexableLoadFunc
                        || (loadFunc instanceof CollectableLoadFunc && loadFunc instanceof OrderedLoadFunc));
                if (combinable)
                    combinable = !conf.getBoolean("pig.noSplitCombination", false);
                JobConf confClone = new JobConf(conf);
                Job inputSpecificJob = new Job(confClone);
                // Pass loader signature to LoadFunc and to InputFormat through
                // the conf
                passLoadSignature(loadFunc, i, inputSpecificJob.getConfiguration());
                loadFunc.setLocation(inputs.get(i).getFileName(),
                        inputSpecificJob);
                // The above setLocation call could write to the conf within
                // the inputSpecificJob - use this updated conf


                // get the InputFormat from it and ask for splits
                InputFormat inpFormat = loadFunc.getInputFormat();
                List<InputSplit> oneInputSplits = inpFormat.getSplits(
                        HadoopShims.createJobContext(inputSpecificJob.getConfiguration(),
                                jobcontext.getJobID()));
                List<InputSplit> oneInputPigSplits = getPigSplits(
                        oneInputSplits, i, inpTargets.get(i),

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.LoadFunc$RequiredField

com.netflix.lipstick.util.OutputSampler

com.twitter.elephantbird.pig.load.MultiFormatLoader

com.twitter.elephantbird.pig.load.TestLocationAsTuple

org.apache.hadoop.fs.Path

org.apache.hadoop.mapreduce.Counter

org.apache.pig.backend.hadoop.executionengine.HJob

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.