Examples of org.apache.hadoop.mapreduce.OutputFormat.checkOutputSpecs()

Class org.apache.hadoop.mapreduce.OutputFormat

Examples of org.apache.hadoop.mapreduce.OutputFormat.checkOutputSpecs()

org.apache.hadoop.mapreduce.OutputFormat.checkOutputSpecs()
Check for validity of the output-specification for the job.
This is to validate the output specification for the job when it is a job is submitted. Typically checks that it does not already exist, throwing an exception when it already exists, so that output is not overwritten.
@param context information about the job @throws IOException when output should not be attempted

            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
            
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            of.checkOutputSpecs(jobContextCopy);
        }
    }
    /**
     * @param currentConf2
     * @param storeLookupKey

View Full Code Here

            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
            
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            of.checkOutputSpecs(jobContextCopy);
        }
    }
    /**
     * @param jobcontext
     * @param storeLookupKey

View Full Code Here

            OutputFormat of = sFunc.getOutputFormat();


            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            try {
                of.checkOutputSpecs(jobContextCopy);
            } catch (IOException ioe) {
                boolean shouldThrowException = true;
                if (sFunc instanceof OverwritableStoreFunc) {
                    if (((OverwritableStoreFunc) sFunc).shouldOverwrite()) {
                        if (ioe instanceof FileAlreadyExistsException

View Full Code Here

      Job job = Job.getInstance(new Configuration());
      JobID jobID = new JobID("200707121733", 1);


      try {
        JobContext context = new JobContext(job.getConfiguration(), jobID);
        outputFormat.checkOutputSpecs(context);
        Assert.fail("No checking for invalid work/commit path");
      } catch (IllegalStateException ignore) { }


      CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
      try {

View Full Code Here

      } catch (IllegalStateException ignore) { }


      CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
      try {
        JobContext context = new JobContext(job.getConfiguration(), jobID);
        outputFormat.checkOutputSpecs(context);
        Assert.fail("No checking for invalid commit path");
      } catch (IllegalStateException ignore) { }


      job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
      CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));

View Full Code Here


      job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
      CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));
      try {
        JobContext context = new JobContext(job.getConfiguration(), jobID);
        outputFormat.checkOutputSpecs(context);
        Assert.fail("No checking for invalid work path");
      } catch (IllegalStateException ignore) { }


      CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
      CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));

View Full Code Here


      CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
      CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));
      try {
        JobContext context = new JobContext(job.getConfiguration(), jobID);
        outputFormat.checkOutputSpecs(context);
      } catch (IllegalStateException ignore) {
        Assert.fail("Output spec check failed.");
      }


    } catch (IOException e) {

View Full Code Here

            OutputFormat of = sFunc.getOutputFormat();
            
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            try {
                of.checkOutputSpecs(jobContextCopy);
            } catch (IOException ioe) {
                boolean shouldThrowException = true;
                if (sFunc instanceof OverwritableStoreFunc) {
                    if (((OverwritableStoreFunc) sFunc).shouldOverwrite()) {
                        if (ioe instanceof FileAlreadyExistsException

View Full Code Here

        throw new IOException(e);
      }


      context.jobContext = jobContext;
      // The contract of the OutputFormat is to check the output specs
      outputFormat.checkOutputSpecs(jobContext);
      // We get the output committer so we can call it later
      context.outputCommitter = outputFormat.getOutputCommitter(taskContext);
      // Save the RecordWriter to cache it
      context.recordWriter = outputFormat.getRecordWriter(taskContext);

View Full Code Here

            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
            
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            of.checkOutputSpecs(jobContextCopy);
        }
    }
    /**
     * @param currentConf2
     * @param storeLookupKey

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.