Package org.apache.hadoop.hive.ql.exec.FileSinkOperator

Examples of org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter

77
78
79
80
81
82
83
84
85
86
87
    final int finalRowSeparator = rowSeparator;
    FileSystem fs = outPath.getFileSystem(jc);
    final OutputStream outStream = Utilities.createCompressedStream(jc, fs
        .create(outPath), isCompressed);
    return new RecordWriter() {
      @Override
      public void write(Writable r) throws IOException {
        if (r instanceof Text) {
          Text tr = (Text) r;
          outStream.write(tr.getBytes(), 0, tr.getLength());
View Full Code Here
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
      throw new HiveException(e);
    }

    for (String p : paths) {
      Path path = new Path(p);
      RecordWriter writer = HiveFileFormatUtils.getRecordWriter(
          jc, hiveOutputFormat, outputClass, isCompressed,
          tableInfo.getProperties(), path, reporter);
      writer.close(false);
      LOG.info("created empty bucket for enforcing bucketing at " + path);
    }
  }
View Full Code Here
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
    //file path.
    newPath = fs.makeQualified(newPath);
    String newFile = newDir + File.separator + "emptyFile";
    Path newFilePath = new Path(newFile);

    RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath,
        Text.class, false, props, null);
    if (dummyRow) {
      // empty files are omitted at CombineHiveInputFormat.
      // for meta-data only query, it effectively makes partition columns disappear..
      // this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955)
      recWriter.write(new Text("empty"));  // written via HiveIgnoreKeyTextOutputFormat
    }
    recWriter.close(false);

    return newPath;
  }
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
        columnMap.put(Bytes.toBytes(columnName), i);
      }
      ++i;
    }

    return new RecordWriter() {

      @Override
      public void close(boolean abort) throws IOException {
        try {
          fileWriter.close(null);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    final boolean walEnabled = HiveConf.getBoolVar(
        jc, HiveConf.ConfVars.HIVE_HBASE_WAL_ENABLED);
    final HTable table = new HTable(new HBaseConfiguration(jc), hbaseTableName);
    table.setAutoFlush(false);

    return new RecordWriter() {

      @Override
      public void close(boolean abort) throws IOException {
        if (!abort) {
          table.flushCommits();
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
        columnMap.put(Bytes.toBytes(columnName), i);
      }
      ++i;
    }

    return new RecordWriter() {

      @Override
      public void close(boolean abort) throws IOException {
        try {
          fileWriter.close(null);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
      Properties tableProperties, Progressable progress) throws IOException {

    FileSystem fs = outPath.getFileSystem(jc);
    final OutputStream outStream = fs.create(outPath, progress);

    return new RecordWriter() {
      @Override
      public void write(Writable r) throws IOException {
        if (r instanceof Text) {
          Text tr = (Text) r;
          outStream.write(tr.getBytes(), 0, tr.getLength());
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc, fs, finalOutPath,
    HiveKey.class, NullWritable.class, isCompressed, progress);

    keyWritable = new HiveKey();
    keyIsText = valueClass.equals(Text.class);
    return new RecordWriter() {
      @Override
      public void write(Writable r) throws IOException {
        if (keyIsText) {
          Text text = (Text) r;
          keyWritable.set(text.getBytes(), 0, text.getLength());
View Full Code Here
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
      throw new HiveException(e);
    }

    for (String p : paths) {
      Path path = new Path(p);
      RecordWriter writer = HiveFileFormatUtils.getRecordWriter(
          jc, hiveOutputFormat, outputClass, isCompressed,
          tableInfo.getProperties(), path, reporter);
      writer.close(false);
      LOG.info("created empty bucket for enforcing bucketing at " + path);
    }
  }
View Full Code Here
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
    //file path.
    newPath = fs.makeQualified(newPath);
    String newFile = newDir + Path.SEPARATOR + "emptyFile";
    Path newFilePath = new Path(newFile);

    RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath,
        Text.class, false, props, null);
    if (dummyRow) {
      // empty files are omitted at CombineHiveInputFormat.
      // for meta-data only query, it effectively makes partition columns disappear..
      // this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955)
      recWriter.write(new Text("empty"));  // written via HiveIgnoreKeyTextOutputFormat
    }
    recWriter.close(false);

    return newPath;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter

  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.facebook.presto.hive.BenchmarkHiveFileFormats
  • org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
  • org.apache.hadoop.hive.hbase.HiveHFileOutputFormat
  • org.apache.hadoop.hive.ql.exec.ExecDriver
  • org.apache.hadoop.hive.ql.exec.Utilities
  • org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
  • org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  • org.apache.hadoop.hive.ql.io.HiveNullValueSequenceFileOutputFormat
  • org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.