Package org.apache.pig

Examples of org.apache.pig.StoreFuncInterface

406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
        if( funcSpec == null ){
            funcSpec = new FuncSpec( PigStorage.class.getName() );
        }
       
        Object obj = PigContext.instantiateFuncFromSpec( funcSpec );
        StoreFuncInterface stoFunc = (StoreFuncInterface)obj;
        String sig = QueryParserUtils.constructSignature( inputAlias, filename, funcSpec);
        stoFunc.setStoreFuncUDFContextSignature( sig );
        String absolutePath = fileNameMap.get( sig );
        if (absolutePath == null) {
            absolutePath = stoFunc.relToAbsPathForStoreLocation( filename,
                    QueryParserUtils.getCurrentDir( pigContext ) );
            if (absolutePath!=null) {
                QueryParserUtils.setHdfsServers( absolutePath, pigContext );
            }
            fileNameMap.put( sig, absolutePath );
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
            TaskAttemptContext context,
            List<POStore> stores) throws IOException {
        List<Pair<OutputCommitter, POStore>> committers =
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
           
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
        return contextCopy;  
    }

    static public void storeCleanup(POStore store, Configuration conf)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()
View Full Code Here
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
            LinkedList<POStore> mapStores = PlanHelper.getPhysicalOperators(mro.mapPlan, POStore.class);
            LinkedList<POStore> reduceStores = PlanHelper.getPhysicalOperators(mro.reducePlan, POStore.class);

            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);
View Full Code Here
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
           
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }

            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);
           
View Full Code Here
TOP

Related Classes of org.apache.pig.StoreFuncInterface

  • org.apache.pig.backend.hadoop.executionengine.fetch.FetchPOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.impl.io.PigFile
  • org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor
  • org.apache.pig.newplan.logical.rules.InputOutputFileValidator$InputOutputFileVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.