Package org.apache.hadoop.mapreduce.lib.output

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputFormat


        }
        VariableTable variables = createVariables(context);
        String destination = path.iterator().next().replace('*', '_');
        String resolved = variables.parse(destination, false);
        Configuration conf = configurations.newInstance();
        FileOutputFormat output = getOpposite(conf, description.getInputFormat());
        FileDeployer deployer = new FileDeployer(conf);
        return deployer.openOutput(definition, resolved, output);
    }
View Full Code Here


                    inputFormat.getName()));
        }
        LOG.debug("Inferred oppsite OutputFormat: {}", outputFormatName);
        try {
            Class<?> loaded = inputFormat.getClassLoader().loadClass(outputFormatName);
            FileOutputFormat instance = (FileOutputFormat) ReflectionUtils.newInstance(loaded, conf);
            if (instance instanceof Configurable) {
                ((Configurable) instance).setConf(conf);
            }
            return instance;
        } catch (Exception e) {
View Full Code Here

        checkType(definition, description);
        VariableTable variables = createVariables(context);
        String destination = description.getPathPrefix().replace('*', '_');
        String resolved = variables.parse(destination, false);
        Configuration conf = configurations.newInstance();
        FileOutputFormat output = ReflectionUtils.newInstance(description.getOutputFormat(), conf);
        FileDeployer deployer = new FileDeployer(conf);
        return deployer.openOutput(definition, resolved, output);
    }
View Full Code Here

    Job job = null;
    JobContext jobContext = null;
    TaskAttemptContext tao = null;
    RecordWriter rw;
    VertexWriter vw;
    FileOutputFormat outputFormat;
   
    boolean tryAgain = true;
    int count = 0;
    while (tryAgain && count < 15)
      try {
        count++;
        tryAgain = false;
        if (job == null) {
          job = new Job(conf);
          job.setOutputFormatClass(TextOutputFormat.class);
          FileOutputFormat.setOutputPath(job, new Path(new String(getOrbConf().getNameNode()
                                                                  + getOrbConf().getFileOutputPath())));
        }
        if (jobContext == null) {
          jobContext = new JobContext(job.getConfiguration(), new JobID());
        }
       
        System.out.println(jobContext.getConfiguration().get("mapred.output.dir"));
       
        tao = new TaskAttemptContext(jobContext.getConfiguration(), new TaskAttemptID(new TaskID(
            jobContext.getJobID(), true, getPartitionID()), 0));
        outputFormat = (FileOutputFormat) tao.getOutputFormatClass().newInstance();
        rw = outputFormat.getRecordWriter(tao);
        vw = (VertexWriter) getOrbConf().getVertexOutputFormatClass().newInstance();
        for (Vertex v : vertices.values()) {
          OrbContext oc = vw.vertexWrite(v);
          rw.write(oc.getKey(), oc.getValue());
          // orbLogger.info("Partition: " + Integer.toString(partitionId) + "writing: " +
          // oc.getKey().toString() + ", " + oc.getValue().toString());
        }
        rw.close(tao);
       
        FileOutputCommitter cm = (FileOutputCommitter) outputFormat.getOutputCommitter(tao);
        if (cm.needsTaskCommit(tao)) {
          cm.commitTask(tao);
          cm.cleanupJob(jobContext);
        } else {
          cm.cleanupJob(jobContext);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.lib.output.FileOutputFormat

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.