Package org.apache.hadoop.mapreduce.lib.output

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

102
103
104
105
106
107
108
109
110
111
112
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here
54
55
56
57
58
59
60
61
  public void checkOutputSpecs(JobContext context)
      throws IOException, InterruptedException {}

  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    Path p1;
    isAppend = context.getConfiguration().get(HIHOConf.IS_APPEND);
    if (isAppend.equalsIgnoreCase("false")) {
      p1 = super.getDefaultWorkFile(context, extension);
    } else {
      FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);
      Path p = committer.getWorkPath();
      fileCount = p.getFileSystem(context.getConfiguration())
          .getContentSummary(getOutputPath(context)).getFileCount();
      if (fileCount > 1) {
        fileCount = fileCount - 1;
      }
      p1 = new Path(committer.getWorkPath(), getUniqueFile(context,
          "part", extension));
    }
    return p1;
  }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    Path p1;
    isAppend = context.getConfiguration().get(HIHOConf.IS_APPEND, "false");
    if (isAppend.equalsIgnoreCase("false")) {
      p1 = super.getDefaultWorkFile(context, extension);
    } else {
      FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);
      Path p = committer.getWorkPath();
      fileCount = p.getFileSystem(context.getConfiguration())
          .getContentSummary(getOutputPath(context)).getFileCount();
      if (fileCount > 1)
        fileCount = fileCount - 1;
      p1 = new Path(committer.getWorkPath(), getUniqueFile(context,
          "part", extension));
    }
    return p1;
  }
View Full Code Here
50
51
52
53
54
55
56
57
  public void checkOutputSpecs(JobContext context)
      throws IOException, InterruptedException {}

  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
            if (components.contains(committer) == false) {
                components.add(committer);
            }
        }
        if (isTemporaryOutputEnabled(context)) {
            FileOutputCommitter committer = temporaryOutputFormat.getOutputCommitter(context);
            if (components.contains(committer) == false) {
                components.add(committer);
            }
        }
        if (LOG.isDebugEnabled()) {
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
        Configuration conf = context.getConfiguration();
        if (FileOutputFormat.getCompressOutput(context)) {
            Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(context, DefaultCodec.class);
            codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        }
        FileOutputCommitter committer = getOutputCommitter(context);
        Path file = new Path(
                committer.getWorkPath(),
                FileOutputFormat.getUniqueFile(context, name, ""));
        final ModelOutput<V> out = TemporaryStorage.openOutput(conf, dataType, file, codec);
        return new RecordWriter<NullWritable, V>() {

            @Override
View Full Code Here
144
145
146
147
148
149
150
151
    private FileOutputCommitter createOutputCommitter(TaskAttemptContext context) throws IOException {
        assert context != null;
        if (getOutputPath(context).equals(FileOutputFormat.getOutputPath(context))) {
            return (FileOutputCommitter) new EmptyFileOutputFormat().getOutputCommitter(context);
        } else {
            return new FileOutputCommitter(getOutputPath(context), context);
        }
    }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here
126
127
128
129
130
131
132
    /** Default implementation assumes FileOutputFormat. Storage drivers wrapping
     * other OutputFormats should override this method.
     */
    public Path getWorkFilePath(TaskAttemptContext context, String outputLoc) throws IOException{
      return new Path(new FileOutputCommitter(new Path(outputLoc), context).getWorkPath(), FileOutputFormat.getUniqueFile(context, "part",""));
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  • co.nubetech.hiho.mapreduce.lib.db.apache.DBOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat
  • com.asakusafw.runtime.stage.output.StageOutputFormat
  • com.asakusafw.runtime.stage.output.TemporaryOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.common.HadoopUtils
  • com.facebook.giraph.hive.impl.common.HadoopUtils
  • com.facebook.hiveio.common.HadoopUtils
  • com.splout.db.hadoop.engine.SploutSQLProxyOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.