Examples of VectorAccessibleSerializable

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

86
87
88
89
90
91
92
93
94
95
96
97
98
    if (outputStream == null) {
      outputStream = fs.create(path);
    }
    int recordCount = newContainer.getRecordCount();
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(recordCount, newContainer, false);
    VectorAccessibleSerializable outputBatch = new VectorAccessibleSerializable(batch, allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    outputBatch.writeToStream(outputStream);
    newContainer.zeroVectors();
//    logger.debug("Took {} us to spill {} records", watch.elapsed(TimeUnit.MICROSECONDS), recordCount);
    spilledBatches++;
  }
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

101
102
103
104
105
106
107
108
109
110
111
112
113
114
    assert fs != null;
    assert path != null;
    if (inputStream == null) {
      inputStream = fs.open(path);
    }
    VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    vas.readFromStream(inputStream);
    VectorContainer c = (VectorContainer) vas.get();
//    logger.debug("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS), c.getRecordCount());
    spilledBatches--;
    return c;
  }
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
        FileSystem fs = FileSystem.get(conf);
      Path path = new Path(filename);
      assertTrue("Trace file does not exist", fs.exists(path));
      FSDataInputStream in = fs.open(path);

      VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
      wrap.readFromStream(in);
      VectorAccessible container = wrap.get();

        /* Assert there are no selection vectors */
      assertTrue(wrap.getSv2() == null);

        /* Assert there is only one record */
        assertTrue(container.getRecordCount() == 1);

        /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
 
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
    } else {
      sv = null;
    }
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(incoming.getRecordCount(), incoming, incomingHasSv2 ? true
        : false);
    VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, sv, oContext.getAllocator());

    try {
      wrap.writeToStreamAndRetain(fos);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    batch.reconstructContainer(container);
    if (incomingHasSv2) {
      sv = wrap.getSv2();
    }
  }
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

170
171
172
173
174
175
176
177
178
179
180
181
182
    List<Integer> schemaChangeIdx = Lists.newArrayList();

    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();

    while (input.available() > 0) {
      VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);
       VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();

       aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));

       if (vectorContainer.getRecordCount() == 0) {
         emptyBatchNum ++;
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
   * @throws Exception
   */
  protected void doBatch(FileInputStream input, int targetBatchNum, boolean showHeader) throws Exception {
    int batchNum = -1;

    VectorAccessibleSerializable vcSerializable = null;

    while (input.available() > 0 && batchNum ++ < targetBatchNum) {
      vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);

      if (batchNum != targetBatchNum) {
        VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
        vectorContainer.zeroVectors();
      }
    }

    if (batchNum < targetBatchNum) {
      System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum+1, batchNum));
      input.close();
      System.exit(-1);
    }

    if (vcSerializable != null) {
      showSingleBatch(vcSerializable, showHeader);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
      vectorContainer.zeroVectors();
    }
  }
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
        FileSystem fs = FileSystem.get(conf);
      Path path = new Path(filename);
      assertTrue("Trace file does not exist", fs.exists(path));
      FSDataInputStream in = fs.open(path);

      VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
      wrap.readFromStream(in);
      VectorAccessible container = wrap.get();

        /* Assert there are no selection vectors */
      assertTrue(wrap.getSv2() == null);

        /* Assert there is only one record */
        assertTrue(container.getRecordCount() == 1);

        /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
 
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

172
173
174
175
176
177
178
179
180
181
182
183
184
    List<Integer> schemaChangeIdx = Lists.newArrayList();

    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();

    while (input.available() > 0) {
      VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();

      aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));

      if (vectorContainer.getRecordCount() == 0) {
        emptyBatchNum ++;
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
   * @throws Exception
   */
  protected void doBatch(FileInputStream input, int targetBatchNum, boolean showHeader) throws Exception {
    int batchNum = -1;

    VectorAccessibleSerializable vcSerializable = null;

    while (input.available() > 0 && batchNum ++ < targetBatchNum) {
      vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);

      if (batchNum != targetBatchNum) {
        VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
        vectorContainer.zeroVectors();
      }
    }

    if (batchNum < targetBatchNum) {
      System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum+1, batchNum));
      input.close();
      System.exit(-1);
    }

    if (vcSerializable != null) {
      showSingleBatch(vcSerializable, showHeader);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
      vectorContainer.zeroVectors();
    }
  }
View Full Code Here

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable

74
75
76
77
78
79
80
81
82
83
84
85
86
    if (outputStream == null) {
      outputStream = fs.create(path);
    }
    int recordCount = newContainer.getRecordCount();
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(recordCount, newContainer, false);
    VectorAccessibleSerializable outputBatch = new VectorAccessibleSerializable(batch, allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    outputBatch.writeToStream(outputStream);
    newContainer.zeroVectors();
    logger.debug("Took {} us to spill {} records", watch.elapsed(TimeUnit.MICROSECONDS), recordCount);
    spilledBatches++;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.