Package org.apache.accumulo.core.file

Examples of org.apache.accumulo.core.file.FileSKVIterator

630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
      Text endRow, TCredentials credentials) throws Exception {
    List<TabletLocation> result = new ArrayList<TabletLocation>();
    Collection<ByteSequence> columnFamilies = Collections.emptyList();
    String filename = file.toString();
    // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
    FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
    try {
      Text row = startRow;
      if (row == null)
        row = new Text();
      while (true) {
        // log.debug(filename + " Seeking to row " + row);
        reader.seek(new Range(row, null), columnFamilies, false);
        if (!reader.hasTop()) {
          // log.debug(filename + " not found");
          break;
        }
        row = reader.getTopKey().getRow();
        TabletLocation tabletLocation = locator.locateTablet(row, false, true, credentials);
        // log.debug(filename + " found row " + row + " at location " + tabletLocation);
        result.add(tabletLocation);
        row = tabletLocation.tablet_extent.getEndRow();
        if (row != null && (endRow == null || row.compareTo(endRow) < 0)) {
          row = new Text(row);
          row.append(byte0, 0, byte0.length);
        } else
          break;
      }
    } finally {
      reader.close();
    }
    // log.debug(filename + " to be sent to " + result);
    return result;
  }
View Full Code Here
538
539
540
541
542
543
544
545
546
       
        list.add(reader);
      }
     
      for (FileDataSource fds : dataSources) {
        FileSKVIterator reader = map.get(fds.file).remove(0);
        fds.setIterator(reader);
      }
    }
View Full Code Here
296
297
298
299
300
301
302
303
304
305
306
    readers.clear();

    // TODO need to close files
    for (String file : absFiles) {
      FileSKVIterator reader = FileOperations.getInstance().openReader(file, false, fs, conf, acuTableConf, null, null);
      readers.add(reader);
    }

    MultiIterator multiIter = new MultiIterator(readers, extent);
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
215
  @Override
  public FileSKVIterator openReader(String file, Range range, Set<ByteSequence> columnFamilies, boolean inclusive, FileSystem fs, Configuration conf,
      AccumuloConfiguration tableConf) throws IOException {
    MyMapFile.Reader mfIter = MapFileUtil.openMapFile(tableConf, fs, file, conf);
   
    FileSKVIterator iter = new RangeIterator(mfIter);
   
    iter.seek(range, columnFamilies, inclusive);
    mfIter.dropIndex();
   
    return iter;
  }
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
      mfw = null; // set this to null so we do not try to close it again in finally if the close fails
      mfwTmp.close(); // if the close fails it will cause the compaction to fail
     
      // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close()
      try {
        FileSKVIterator openReader = fileFactory.openReader(outputFile, false, fs, conf, tableConf);
        openReader.close();
      } catch (IOException ex) {
        log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile, ex);
        throw ex;
      }
     
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    for (String mapFile : filesToCompact.keySet()) {
      try {
       
        FileOperations fileFactory = FileOperations.getInstance();
       
        FileSKVIterator reader;
       
        AccumuloConfiguration tableConf = ServerConfiguration.getTableConfiguration(extent.getTableId().toString());
       
        reader = fileFactory.openReader(mapFile, false, fs, conf, tableConf);
       
        readers.add(reader);
       
        SortedKeyValueIterator<Key,Value> iter = new ProblemReportingIterator(extent.getTableId().toString(), mapFile, false, reader);
       
        if (filesToCompact.get(mapFile).isTimeSet()) {
          iter = new TimeSettingIterator(iter, filesToCompact.get(mapFile).getTime());
        }
       
        iters.add(iter);
       
      } catch (Throwable e) {
       
        ProblemReports.getInstance().report(new ProblemReport(extent.getTableId().toString(), ProblemType.FILE_READ, mapFile, e));
       
        log.warn("Some problem opening map file " + mapFile + " " + e.getMessage(), e);
        // failed to open some map file... close the ones that were opened
        for (FileSKVIterator reader : readers) {
          try {
            reader.close();
          } catch (Throwable e2) {
            log.warn("Failed to close map file", e2);
          }
        }
       
View Full Code Here
61
62
63
64
65
66
67
68
69
70
  private Range range = new Range();

  private List<SortedKeyValueIterator<Key,Value>> openMapFiles(Collection<String> files, FileSystem fs, Configuration conf) throws IOException {
    List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>();
    for (String file : files) {
      FileSKVIterator reader = FileOperations.getInstance().openReader(file, true, fs, conf, ServerConfiguration.getSystemConfiguration());
      readers.add(reader);
    }
    return readers;
  }
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
        else {
         
          Configuration conf = CachedConfiguration.getInstance();
          FileSystem fs = TraceFileSystem.wrap(FileSystem.getLocal(conf));
         
          @SuppressWarnings("deprecation")
          FileSKVIterator reader = new MapFileOperations.RangeIterator(new MyMapFile.Reader(fs, memDumpFile, conf));
         
          readers.add(reader);
         
          iter = reader;
View Full Code Here
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
      Text endRow) throws Exception {
    List<TabletLocation> result = new ArrayList<TabletLocation>();
    Collection<ByteSequence> columnFamilies = Collections.emptyList();
    String filename = file.toString();
    // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
    FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
    try {
      Text row = startRow;
      if (row == null)
        row = new Text();
      while (true) {
        // log.debug(filename + " Seeking to row " + row);
        reader.seek(new Range(row, null), columnFamilies, false);
        if (!reader.hasTop()) {
          // log.debug(filename + " not found");
          break;
        }
        row = reader.getTopKey().getRow();
        TabletLocation tabletLocation = locator.locateTablet(row, false, true);
        // log.debug(filename + " found row " + row + " at location " + tabletLocation);
        result.add(tabletLocation);
        row = tabletLocation.tablet_extent.getEndRow();
        if (row != null && (endRow == null || row.compareTo(endRow) < 0)) {
          row = new Text(row);
          row.append(byte0, 0, byte0.length);
        } else
          break;
      }
    } finally {
      reader.close();
    }
    // log.debug(filename + " to be sent to " + result);
    return result;
  }
View Full Code Here
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
     
      long rtime = Long.MIN_VALUE;
      for (String path : datafiles.keySet()) {
        String filename = new Path(path).getName();
       
        FileSKVIterator reader = FileOperations.getInstance().openReader(this.location + "/" + filename, true, fs, fs.getConf(),
            ServerConfiguration.getTableConfiguration(Constants.METADATA_TABLE_ID));
        long maxTime = -1;
        try {
         
          while (reader.hasTop()) {
            maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
            reader.next();
          }
         
        } finally {
          reader.close();
        }
       
        if (maxTime > rtime) {
          time = TabletTime.LOGICAL_TIME_ID + "" + maxTime;
          rtime = maxTime;
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.file.FileSKVIterator

  • org.apache.accumulo.core.client.impl.OfflineIterator
  • org.apache.accumulo.core.client.mock.MockTableOperations
  • org.apache.accumulo.core.client.mock.MockTableOperationsImpl
  • org.apache.accumulo.core.file.map.MapFileOperations
  • org.apache.accumulo.core.file.rfile.RFileOperations
  • org.apache.accumulo.core.file.rfile.RFileTest
  • org.apache.accumulo.core.util.BulkImportHelper
  • org.apache.accumulo.server.client.BulkImporter
  • org.apache.accumulo.server.tabletserver.Compactor
  • org.apache.accumulo.server.tabletserver.FileManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.