Package org.apache.hadoop.hive.ql.exec.FileSinkOperator

Examples of org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter

725
726
727
728
729
730
731
732
733
734
      pathToPartitionInfo.put(newPath.toUri().toString(), pDesc);
    }
    work.setPathToPartitionInfo(pathToPartitionInfo);

    String onefile = newPath.toString();
    RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newPath, Text.class, false, new Properties(), null);
    recWriter.close(false);
    FileInputFormat.addInputPaths(job, onefile);
    return numEmptyPaths;
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    FileSystem fs = finalOutPath.getFileSystem(jc);
    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
        fs, finalOutPath, BytesWritable.class, valueClass,
        isCompressed);

    return new RecordWriter() {
      public void write(Writable r) throws IOException {
        outStream.append(EMPTY_KEY, r);
      }

      public void close(boolean abort) throws IOException {
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    final int finalRowSeparator = rowSeparator;
    FileSystem fs = outPath.getFileSystem(jc);
    final OutputStream outStream = Utilities.createCompressedStream(jc,
        fs.create(outPath), isCompressed);
    return new RecordWriter() {
      public void write(Writable r) throws IOException {
        if (r instanceof Text) {
          Text tr = (Text) r;
          outStream.write(tr.getBytes(), 0, tr.getLength());
          outStream.write(finalRowSeparator);
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
    final boolean walEnabled = HiveConf.getBoolVar(
        jc, HiveConf.ConfVars.HIVE_HBASE_WAL_ENABLED);
    final HTable table = new HTable(new HBaseConfiguration(jc), hbaseTableName);
    table.setAutoFlush(false);

    return new RecordWriter() {

      @Override
      public void close(boolean abort) throws IOException {
        if (!abort) {
          table.flushCommits();
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
        columnMap.put(Bytes.toBytes(columnName), i);
      }
      ++i;
    }

    return new RecordWriter() {

      @Override
      public void close(boolean abort) throws IOException {
        try {
          fileWriter.close(null);
View Full Code Here
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
      throw new HiveException(e);
    }

    for (String p : paths) {
      Path path = new Path(p);
      RecordWriter writer = HiveFileFormatUtils.getRecordWriter(
          jc, hiveOutputFormat, outputClass, isCompressed, tableInfo.getProperties(), path);
      writer.close(false);
      LOG.info("created empty bucket for enforcing bucketing at " + path);
    }
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    FileSystem fs = finalOutPath.getFileSystem(jc);
    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
        fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);

    return new RecordWriter() {
      public void write(Writable r) throws IOException {
        outStream.append(EMPTY_KEY, r);
      }

      public void close(boolean abort) throws IOException {
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    final int finalRowSeparator = rowSeparator;
    FileSystem fs = outPath.getFileSystem(jc);
    final OutputStream outStream = Utilities.createCompressedStream(jc, fs
        .create(outPath), isCompressed);
    return new RecordWriter() {
      public void write(Writable r) throws IOException {
        if (r instanceof Text) {
          Text tr = (Text) r;
          outStream.write(tr.getBytes(), 0, tr.getLength());
          outStream.write(finalRowSeparator);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
        fs, finalOutPath, HiveKey.class, NullWritable.class, isCompressed);

    keyWritable = new HiveKey();
    keyIsText = valueClass.equals(Text.class);
    return new RecordWriter() {
      public void write(Writable r) throws IOException {
        if (keyIsText) {
          Text text = (Text) r;
          keyWritable.set(text.getBytes(), 0, text.getLength());
        } else {
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
      Properties tableProperties, Progressable progress) throws IOException {

    FileSystem fs = outPath.getFileSystem(jc);
    final OutputStream outStream = fs.create(outPath);

    return new RecordWriter() {
      public void write(Writable r) throws IOException {
        if (r instanceof Text) {
          Text tr = (Text) r;
          outStream.write(tr.getBytes(), 0, tr.getLength());
        } else {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter

  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.facebook.presto.hive.BenchmarkHiveFileFormats
  • org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
  • org.apache.hadoop.hive.hbase.HiveHFileOutputFormat
  • org.apache.hadoop.hive.ql.exec.ExecDriver
  • org.apache.hadoop.hive.ql.exec.Utilities
  • org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
  • org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  • org.apache.hadoop.hive.ql.io.HiveNullValueSequenceFileOutputFormat
  • org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.