Package org.apache.hcatalog.mapreduce

Examples of org.apache.hcatalog.mapreduce.HCatOutputFormat

61
62
63
64
65
66
67
    this(null,null);
  }

  @Override
  public OutputFormat getOutputFormat() throws IOException {
    return new HCatOutputFormat();
  }
View Full Code Here
850
851
852
853
854
855
856
857
  }

  public void invokeOutputCommitterForLocalMode(Job job) throws IOException {
    if (isLocalJobTracker(job) && isHadoop1()) {
      LOG.info("Explicitly committing job in local mode");
      HCatHadoopShims.Instance.get().commitJob(new HCatOutputFormat(), job);
    }
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
        Job job;
        try {
            job = new Job(conf);
            HCatOutputFormat.setOutput(job, jobInfo);
            HCatOutputFormat.setSchema(job, HCatOutputFormat.getTableSchema(job));
            HCatOutputFormat outFormat = new HCatOutputFormat();
            outFormat.checkOutputSpecs(job);
            outFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
                (job.getConfiguration(), HCatHadoopShims.Instance.get().createTaskAttemptID())).setupJob(job);
        } catch (IOException e) {
            throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
        } catch (InterruptedException e) {
            throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    @Override
    public void write(Iterator<HCatRecord> recordItr) throws HCatException {

        int id = sp.getId();
        setVarsInConf(id);
        HCatOutputFormat outFormat = new HCatOutputFormat();
        TaskAttemptContext cntxt = HCatHadoopShims.Instance.get().createTaskAttemptContext
            (conf, new TaskAttemptID(HCatHadoopShims.Instance.get().createTaskID(), id));
        OutputCommitter committer = null;
        RecordWriter<WritableComparable<?>, HCatRecord> writer;
        try {
            committer = outFormat.getOutputCommitter(cntxt);
            committer.setupTask(cntxt);
            writer = outFormat.getRecordWriter(cntxt);
            while (recordItr.hasNext()) {
                HCatRecord rec = recordItr.next();
                writer.write(null, rec);
            }
            writer.close(cntxt);
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
    }

    @Override
    public void commit(WriterContext context) throws HCatException {
        try {
            new HCatOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
                (context.getConf(), HCatHadoopShims.Instance.get().createTaskAttemptID()))
                .commitJob(HCatHadoopShims.Instance.get().createJobContext(context.getConf(), null));
        } catch (IOException e) {
            throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
        } catch (InterruptedException e) {
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
    }

    @Override
    public void abort(WriterContext context) throws HCatException {
        try {
            new HCatOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
                (context.getConf(), HCatHadoopShims.Instance.get().createTaskAttemptID()))
                .abortJob(HCatHadoopShims.Instance.get().createJobContext(context.getConf(), null), State.FAILED);
        } catch (IOException e) {
            throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
        } catch (InterruptedException e) {
View Full Code Here
71
72
73
74
75
76
77
        this(null, null);
    }

    @Override
    public OutputFormat getOutputFormat() throws IOException {
        return new HCatOutputFormat();
    }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
    Job job;
    try {
      job = new Job(conf);
      HCatOutputFormat.setOutput(job, jobInfo);
      HCatOutputFormat.setSchema(job, HCatOutputFormat.getTableSchema(job));
      HCatOutputFormat outFormat = new HCatOutputFormat();
      outFormat.checkOutputSpecs(job);
      outFormat.getOutputCommitter(new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID())).setupJob(job);
    } catch (IOException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
    } catch (InterruptedException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
    }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
  @Override
  public void write(Iterator<HCatRecord> recordItr) throws HCatException {
   
    int id = sp.getId();
    setVarsInConf(id);
    HCatOutputFormat outFormat = new HCatOutputFormat();
    TaskAttemptContext cntxt = new TaskAttemptContext(conf, new TaskAttemptID(new TaskID(), id));
    OutputCommitter committer = null;
    RecordWriter<WritableComparable<?>, HCatRecord> writer;
    try {
      committer = outFormat.getOutputCommitter(cntxt);
      committer.setupTask(cntxt);
      writer   = outFormat.getRecordWriter(cntxt);
      while(recordItr.hasNext()){
        HCatRecord rec = recordItr.next();
        writer.write(null, rec);
      }
      writer.close(cntxt);
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
  }

  @Override
  public void commit(WriterContext context) throws HCatException {
    try {
      new HCatOutputFormat().getOutputCommitter(new TaskAttemptContext(context.getConf(), new TaskAttemptID()))
      .commitJob(new JobContext(context.getConf(), null));
    } catch (IOException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
    } catch (InterruptedException e) {
      throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.mapreduce.HCatOutputFormat

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileStatus
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hive.conf.HiveConf
  • org.apache.hadoop.hive.metastore.api.Index
  • org.apache.hadoop.hive.metastore.api.StorageDescriptor
  • org.apache.hadoop.hive.metastore.api.Table
  • org.apache.hadoop.hive.metastore.HiveMetaStoreClient
  • org.apache.hadoop.hive.ql.metadata.Table
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.