Package org.apache.accumulo.core.client.impl

Examples of org.apache.accumulo.core.client.impl.ScannerImpl

1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
            datafilesMetadata.put(new Key(entry.getKey()), new Value(entry.getValue()));
          }
        }
      } else {
       
        ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID,
            Constants.NO_AUTHS);
       
        // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
        // reduced batch size to improve performance
        // changed here after endKeys were implemented from 10 to 1000
        mdScanner.setBatchSize(1000);
       
        // leave these in, again, now using endKey for safety
        mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
       
        mdScanner.setRange(new Range(rowName));
       
        datafilesMetadata = new TreeMap<Key,Value>();
       
        for (Entry<Key,Value> entry : mdScanner) {
         
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
  }
 
  public static SortedMap<String,DataFileValue> getDataFileSizes(KeyExtent extent, AuthInfo credentials) {
    TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
   
    Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    mdScanner.setRange(Constants.METADATA_KEYSPACE);
    mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
    Text row = extent.getMetadataEntry();
   
    Key endKey = new Key(row, Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(""));
    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
   
    mdScanner.setRange(new Range(new Key(row), endKey));
    for (Entry<Key,Value> entry : mdScanner) {
     
      if (!entry.getKey().getRow().equals(row))
        break;
      DataFileValue dfv = new DataFileValue(entry.getValue().get());
View Full Code Here
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
   
    Text prevPrevEndRow = KeyExtent.decodePrevEndRow(oper);
   
    low = new KeyExtent(table, metadataPrevEndRow, prevPrevEndRow);
   
    Scanner scanner3 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    Key rowKey = new Key(metadataEntry);
   
    SortedMap<String,DataFileValue> origDatafileSizes = new TreeMap<String,DataFileValue>();
    SortedMap<String,DataFileValue> highDatafileSizes = new TreeMap<String,DataFileValue>();
    SortedMap<String,DataFileValue> lowDatafileSizes = new TreeMap<String,DataFileValue>();
    scanner3.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
    scanner3.setRange(new Range(rowKey, rowKey.followingKey(PartialKey.ROW)));
   
    for (Entry<Key,Value> entry : scanner3) {
      if (entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
        origDatafileSizes.put(entry.getKey().getColumnQualifier().toString(), new DataFileValue(entry.getValue().get()));
      }
    }
   
    splitDatafiles(table, metadataPrevEndRow, splitRatio, new HashMap<String,FileUtil.FileInfo>(), origDatafileSizes, lowDatafileSizes, highDatafileSizes,
        highDatafilesToRemove);
   
    // check to see if prev tablet exist in metadata tablet
    Key prevRowKey = new Key(new Text(KeyExtent.getMetadataEntry(table, metadataPrevEndRow)));
   
    ScannerImpl scanner2 = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    scanner2.setRange(new Range(prevRowKey, prevRowKey.followingKey(PartialKey.ROW)));
   
    if (!scanner2.iterator().hasNext()) {
      log.debug("Prev tablet " + prevRowKey + " does not exist, need to create it " + metadataPrevEndRow + " " + prevPrevEndRow + " " + splitRatio);
      Map<String,Long> bulkFiles = getBulkFilesLoaded(credentials, metadataEntry);
      MetadataTable.addNewTablet(low, lowDirectory, tserver, lowDatafileSizes, bulkFiles, credentials, time, initFlushID, initCompactID, lock);
    } else {
      log.debug("Prev tablet " + prevRowKey + " exist, do not need to add it");
View Full Code Here
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
   
    return fixSplit(table, metadataEntry, metadataPrevEndRow, oper, splitRatio, tserver, credentials, time.toString(), initFlushID, initCompactID, lock);
  }
 
  public static void deleteTable(String tableId, boolean insertDeletes, AuthInfo credentials, ZooLock lock) throws AccumuloException {
    Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    Text tableIdText = new Text(tableId);
    BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, 1000000, 120000l, 2);
   
    // scan metadata for our table and delete everything we find
    Mutation m = null;
    ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());
   
    // insert deletes before deleting data from !METADATA... this makes the code fault tolerant
    if (insertDeletes) {
     
      ms.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
      ColumnFQ.fetch(ms, Constants.METADATA_DIRECTORY_COLUMN);
     
      for (Entry<Key,Value> cell : ms) {
        Key key = cell.getKey();
       
        if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
          String relPath = key.getColumnQualifier().toString();
          // only insert deletes for files owned by this table
          if (!relPath.startsWith("../"))
            bw.addMutation(createDeleteMutation(tableId, relPath));
        }
       
        if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
          bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
        }
      }
     
      bw.flush();
     
      ms.clearColumns();
    }
   
    for (Entry<Key,Value> cell : ms) {
      Key key = cell.getKey();
     
View Full Code Here
825
826
827
828
829
830
831
832
833
834
835
836
837
838
        DataFileValue dfv = new DataFileValue(0, 0);
        sizes.put(Constants.ZROOT_TABLET + "/" + fileStatus.getPath().getName(), dfv);
      }
     
    } else {
      Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
      scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
      scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
      scanner.setRange(extent.toMetadataRange());
     
      for (Entry<Key,Value> entry : scanner) {
        if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) {
          throw new RuntimeException("Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry());
        }
View Full Code Here
895
896
897
898
899
900
901
902
903
904
905
906
907
      result.add(e);
    }
  }
 
  private static Scanner getTabletLogScanner(AuthInfo credentials, KeyExtent extent) {
    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
    Text start = extent.getMetadataEntry();
    Key endKey = new Key(start, Constants.METADATA_LOG_COLUMN_FAMILY);
    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
    scanner.setRange(new Range(new Key(start), endKey));
    return scanner;
  }
View Full Code Here
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
 
  public static Map<String,Long> getBulkFilesLoaded(AuthInfo credentials, Text metadataRow) {
   
    Map<String,Long> ret = new HashMap<String,Long>();
   
    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    scanner.setRange(new Range(metadataRow));
    scanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
    for (Entry<Key,Value> entry : scanner) {
      String file = entry.getKey().getColumnQualifier().toString();
      Long tid = Long.parseLong(entry.getValue().toString());
     
      ret.put(file, tid);
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
      ensureTabletHasNoUnexpectedMetadataEntries(extent, mapFiles);
    }
  }
 
  private void ensureTabletHasNoUnexpectedMetadataEntries(KeyExtent extent, SortedMap<String,DataFileValue> expectedMapFiles) throws Exception {
    Scanner scanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID,
        Constants.NO_AUTHS);
    scanner.setRange(extent.toMetadataRange());
   
    HashSet<ColumnFQ> expectedColumns = new HashSet<ColumnFQ>();
    expectedColumns.add(Constants.METADATA_DIRECTORY_COLUMN);
    expectedColumns.add(Constants.METADATA_PREV_ROW_COLUMN);
    expectedColumns.add(Constants.METADATA_TIME_COLUMN);
    expectedColumns.add(Constants.METADATA_LOCK_COLUMN);
   
    HashSet<Text> expectedColumnFamilies = new HashSet<Text>();
    expectedColumnFamilies.add(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
    expectedColumnFamilies.add(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
    expectedColumnFamilies.add(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
    expectedColumnFamilies.add(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
    expectedColumnFamilies.add(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
   
    Iterator<Entry<Key,Value>> iter = scanner.iterator();
    while (iter.hasNext()) {
      Key key = iter.next().getKey();
     
      if (!key.getRow().equals(extent.getMetadataEntry())) {
        throw new Exception("Tablet " + extent + " contained unexpected " + Constants.METADATA_TABLE_NAME + " entry " + key);
View Full Code Here
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
     
      SortedMap<Key,Value> datafilesMetadata;
     
      Text rowName = extent.getMetadataEntry();
       
      ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID,
          Constants.NO_AUTHS);
     
      // Commented out because when no data file is present, each tablet will scan through metadata table and return nothing
      // reduced batch size to improve performance
      // changed here after endKeys were implemented from 10 to 1000
      mdScanner.setBatchSize(1000);
     
      // leave these in, again, now using endKey for safety
      mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
     
      mdScanner.setRange(new Range(rowName));
     
      datafilesMetadata = new TreeMap<Key,Value>();
     
      for (Entry<Key,Value> entry : mdScanner) {
       
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
  }
 
  public static SortedMap<String,DataFileValue> getDataFileSizes(KeyExtent extent, TCredentials credentials) {
    TreeMap<String,DataFileValue> sizes = new TreeMap<String,DataFileValue>();
   
    Scanner mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
    Text row = extent.getMetadataEntry();
   
    Key endKey = new Key(row, Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(""));
    endKey = endKey.followingKey(PartialKey.ROW_COLFAM);
   
    mdScanner.setRange(new Range(new Key(row), endKey));
    for (Entry<Key,Value> entry : mdScanner) {
     
      if (!entry.getKey().getRow().equals(row))
        break;
      DataFileValue dfv = new DataFileValue(entry.getValue().get());
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.client.impl.ScannerImpl

  • org.apache.accumulo.core.client.mapred.AbstractInputFormat$AbstractRecordReader
  • org.apache.accumulo.core.client.mapreduce.AbstractInputFormat$AbstractRecordReader
  • org.apache.accumulo.core.data.Range
  • org.apache.accumulo.core.util.MetadataTable
  • org.apache.accumulo.server.gc.SimpleGarbageCollector
  • org.apache.accumulo.server.tabletserver.Tablet
  • org.apache.accumulo.server.tabletserver.TabletServer
  • org.apache.accumulo.server.util.CheckForMetadataProblems
  • org.apache.accumulo.server.util.MasterMetadataUtil
  • org.apache.accumulo.server.util.MetadataTable
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.