Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.OutputFormat

51
52
53
54
55
56
57
58
59
60
61
62
    }
  }
 
  public OrcFileWriter(Configuration conf, Path path, PType<T> pType) throws IOException {
    JobConf jobConf = new JobConf(conf);
    OutputFormat outputFormat = new OrcOutputFormat();
    writer = outputFormat.getRecordWriter(null, jobConf, path.toString(), new NullProgress());
   
    mapFn = pType.getOutputMapFn();
    mapFn.initialize();
   
    serde = new OrcSerde();
View Full Code Here
616
617
618
619
620
621
622
623
     if (isMapOnly)
       AvroJob.setMapOutputSchema(outputConf, schema);
     else
       AvroJob.setOutputSchema(outputConf, schema);
   }
   OutputFormat outputFormat = outputConf.getOutputFormat();
   return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3);
   }  
View Full Code Here
537
538
539
540
541
542
543
544
     if (isMapOnly)
       AvroJob.setMapOutputSchema(outputConf, schema);
     else
       AvroJob.setOutputSchema(outputConf, schema);
   }
   OutputFormat outputFormat = outputConf.getOutputFormat();
   return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3);
   }  
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
            confCopy.set("pig.storeFunc", ObjectSerializer.serialize(
                    store.getSFile().getFuncSpec().toString()));
            confCopy.set(JobControlCompiler.PIG_STORE_CONFIG,
                    ObjectSerializer.serialize(storeConfig));
            confCopy.setOutputFormat(sPrepClass);
            OutputFormat of = confCopy.getOutputFormat();
            of.checkOutputSpecs(fs, confCopy);
       
        }
       
    }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
        // The workpath is set to a unique-per-store subdirectory of
        // the current working directory.
        String workPath = outputConf.get("mapred.work.output.dir");
        outputConf.set("mapred.work.output.dir",
                       new Path(workPath, tmpPath).toString());
        OutputFormat outputFormat = outputConf.getOutputFormat();

        // Generate a unique part name (part-<task_partition_number>).
        String fileName = getPartName(outputConf);
       
        // create a new record writer
        writer = outputFormat.getRecordWriter(FileSystem.get(outputConf),
                                              outputConf, fileName, reporter);

        // return an output collector using the writer we just created.
        return new StoreFuncAdaptor(new OutputCollector()
            {
View Full Code Here
537
538
539
540
541
542
543
544
     if (isMapOnly)
       AvroJob.setMapOutputSchema(outputConf, schema);
     else
       AvroJob.setOutputSchema(outputConf, schema);
   }
   OutputFormat outputFormat = outputConf.getOutputFormat();
   return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3);
   }  
View Full Code Here
533
534
535
536
537
538
539
540
   Schema schema = schemaList.get(nameOutput+"_SCHEMA");
   JobConf outputConf = new JobConf(job);
   outputConf.setOutputFormat(getNamedOutputFormatClass(job, nameOutput));
   if(schema!=null)
    AvroJob.setOutputSchema(outputConf,schema);
   OutputFormat outputFormat = outputConf.getOutputFormat();
   return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3);
   }  
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        // The workpath is set to a unique-per-store subdirectory of
        // the current working directory.
        String workPath = outputConf.get("mapred.work.output.dir");
        outputConf.set("mapred.work.output.dir",
                       new Path(workPath, tmpPath).toString());
        OutputFormat outputFormat = outputConf.getOutputFormat();

        // Generate a unique part name (part-<task_partition_number>).
        String fileName = getPartName(outputConf);
       
        // create a new record writer
        writer = outputFormat.getRecordWriter(FileSystem.get(outputConf),
                                              outputConf, fileName, reporter);

        // return an output collector using the writer we just created.
        return new StoreFuncAdaptor(new OutputCollector()
            {
View Full Code Here
619
620
621
622
623
624
625
626
     if (isMapOnly)
       AvroJob.setMapOutputSchema(outputConf, schema);
     else
       AvroJob.setOutputSchema(outputConf, schema);
   }
   OutputFormat outputFormat = outputConf.getOutputFormat();
   return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3);
   }  
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
        this.attempt = attempt;

        if (directWrite) {
            jobConf.set("mapreduce.task.attempt.id", attempt.toString());

            OutputFormat outFormat = jobConf.getOutputFormat();

            writer = outFormat.getRecordWriter(null, jobConf, fileName, Reporter.NULL);
        }
        else
            writer = null;
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.OutputFormat

  • cascading.tap.hadoop.io.TapOutputCollector
  • org.apache.avro.mapred.AvroMultipleOutputs$InternalFileOutputFormat
  • org.apache.crunch.io.orc.OrcFileWriter
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat
  • org.gridgain.grid.kernal.processors.hadoop.v1.GridHadoopV1OutputCollector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.