Package org.apache.accumulo.core.file.rfile.RFile

Examples of org.apache.accumulo.core.file.rfile.RFile.Reader

64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    for (String arg : opts.files) {
     
      Path path = new Path(arg);
      FileSystem fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
      CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null);
      Reader iter = new RFile.Reader(_rdr);
     
      iter.printInfo();
      System.out.println();
      org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] {arg});
     
      if (opts.histogram || opts.dump) {
        iter.seek(new Range((Key) null, (Key) null), new ArrayList<ByteSequence>(), false);
        while (iter.hasTop()) {
          Key key = iter.getTopKey();
          Value value = iter.getTopValue();
          if (opts.dump)
            System.out.println(key + " -> " + value);
          if (opts.histogram) {
            long size = key.getSize() + value.getSize();
            int bucket = (int) Math.log10(size);
            countBuckets[bucket]++;
            sizeBuckets[bucket] += size;
            totalSize += size;
          }
          iter.next();
        }
      }
      iter.close();
      if (opts.histogram) {
        System.out.println("Up to size      count      %-age");
        for (int i = 1; i < countBuckets.length; i++) {
          System.out.println(String.format("%11.0f : %10d %6.2f%%", Math.pow(10, i), countBuckets[i], sizeBuckets[i] * 100. / totalSize));
        }
View Full Code Here
58
59
60
61
62
63
64
65
66
    Path path = new Path(file);
    // long len = fs.getFileStatus(path).getLen();
    // FSDataInputStream in = fs.open(path);
    // Reader reader = new RFile.Reader(in, len , conf);
    CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, dataCache, indexCache);
    final Reader reader = new RFile.Reader(_cbr);
   
    return reader.getIndex();
  }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
  public FileSKVIterator openReader(String file, boolean seekToBeginning, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf,
      BlockCache dataCache, BlockCache indexCache) throws IOException {
    Path path = new Path(file);
   
    CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, dataCache, indexCache);
    Reader iter = new RFile.Reader(_cbr);
   
    if (seekToBeginning) {
      iter.seek(new Range((Key) null, null), EMPTY_CF_SET, false);
    }
   
    return iter;
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    for (String arg : commandLine.getArgs()) {
     
      Path path = new Path(arg);
      FileSystem fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
      CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null);
      Reader iter = new RFile.Reader(_rdr);
     
      iter.printInfo();
      System.out.println();
      org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] {arg});
     
      if (doHistogram || dump) {
        iter.seek(new Range((Key) null, (Key) null), new ArrayList<ByteSequence>(), false);
        while (iter.hasTop()) {
          Key key = iter.getTopKey();
          Value value = iter.getTopValue();
          if (dump)
            System.out.println(key + " -> " + value);
          if (doHistogram) {
            long size = key.getSize() + value.getSize();
            int bucket = (int) Math.log10(size);
            countBuckets[bucket]++;
            sizeBuckets[bucket] += size;
            totalSize += size;
          }
          iter.next();
        }
      }
      iter.close();
      if (doHistogram) {
        System.out.println("Up to size      count      %-age");
        for (int i = 1; i < countBuckets.length; i++) {
          System.out.println(String.format("%11.0f : %10d %6.2f%%", Math.pow(10, i), countBuckets[i], sizeBuckets[i] * 100. / totalSize));
        }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    opts.parseArgs(SplitLarge.class.getName(), args);
   
    for (String file : opts.files) {
      Path path = new Path(file);
      CachableBlockFile.Reader rdr = new CachableBlockFile.Reader(fs, path, conf, null, null);
      Reader iter = new RFile.Reader(rdr);
     
      if (!file.endsWith(".rf")) {
        throw new IllegalArgumentException("File must end with .rf");
      }
      String smallName = file.substring(0, file.length() - 3) + "_small.rf";
      String largeName = file.substring(0, file.length() - 3) + "_large.rf";
     
      int blockSize = (int) DefaultConfiguration.getDefaultConfiguration().getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE);
      Writer small = new RFile.Writer(new CachableBlockFile.Writer(fs, new Path(smallName), "gz", conf), blockSize);
      small.startDefaultLocalityGroup();
      Writer large = new RFile.Writer(new CachableBlockFile.Writer(fs, new Path(largeName), "gz", conf), blockSize);
      large.startDefaultLocalityGroup();

      iter.seek(new Range(), new ArrayList<ByteSequence>(), false);
      while (iter.hasTop()) {
        Key key = iter.getTopKey();
        Value value = iter.getTopValue();
        if (key.getSize() + value.getSize() < maxSize) {
          small.append(key, value);
        } else {
          large.append(key, value);
        }
        iter.next();
      }

      iter.close();
      large.close();
      small.close();
    }
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
        log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
        fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
      }
     
      CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf);
      Reader iter = new RFile.Reader(_rdr);
     
      iter.printInfo();
      System.out.println();
      org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] {arg});
     
      if (opts.histogram || opts.dump) {
        iter.seek(new Range((Key) null, (Key) null), new ArrayList<ByteSequence>(), false);
        while (iter.hasTop()) {
          Key key = iter.getTopKey();
          Value value = iter.getTopValue();
          if (opts.dump)
            System.out.println(key + " -> " + value);
          if (opts.histogram) {
            long size = key.getSize() + value.getSize();
            int bucket = (int) Math.log10(size);
            countBuckets[bucket]++;
            sizeBuckets[bucket] += size;
            totalSize += size;
          }
          iter.next();
        }
      }
      iter.close();
      if (opts.histogram) {
        System.out.println("Up to size      count      %-age");
        for (int i = 1; i < countBuckets.length; i++) {
          System.out.println(String.format("%11.0f : %10d %6.2f%%", Math.pow(10, i), countBuckets[i], sizeBuckets[i] * 100. / totalSize));
        }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
   
    for (String arg : commandLine.getArgs()) {
     
      Path path = new Path(arg);
      CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null);
      Reader iter = new RFile.Reader(_rdr);
     
      iter.printInfo();
     
      if (commandLine.hasOption(dumpKeys.getOpt())) {
        iter.seek(new Range((Key) null, (Key) null), new ArrayList<ByteSequence>(), false);
        while (iter.hasTop()) {
          Key key = iter.getTopKey();
          Value value = iter.getTopValue();
          System.out.println(key + " -> " + value);
          iter.next();
        }
      }
     
      iter.close();
    }
  }
View Full Code Here
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
   
    byte data[] = baos.toByteArray();
    SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
    FSDataInputStream in2 = new FSDataInputStream(bais);
    CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in2, data.length, CachedConfiguration.getInstance());
    Reader reader = new RFile.Reader(_cbr);
    checkIndex(reader);
   
    ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader);
   
    for (int start : new int[] {0, 10, 100, 998}) {
      for (int cf = 1; cf <= 4; cf++) {
        if (start == 0)
          iter.seek(new Range(), ncfs(nf("cf_", cf)), true);
        else
          iter.seek(new Range(nf("r_", start), null), ncfs(nf("cf_", cf)), true);
       
        for (int i = start; i < 1000; i++) {
          assertTrue(iter.hasTop());
          assertEquals(nk(nf("r_", i), nf("cf_", cf), nf("cq_", 0), "", 1000 - i), iter.getTopKey());
          assertEquals(nv(i + ""), iter.getTopValue());
          iter.next();
        }
       
        assertFalse(iter.hasTop());
      }
     
      if (start == 0)
        iter.seek(new Range(), ncfs(), false);
      else
        iter.seek(new Range(nf("r_", start), null), ncfs(), false);
     
      for (int i = start; i < 1000; i++) {
        for (int cf = 1; cf <= 4; cf++) {
          assertTrue(iter.hasTop());
          assertEquals(nk(nf("r_", i), nf("cf_", cf), nf("cq_", 0), "", 1000 - i), iter.getTopKey());
          assertEquals(nv(i + ""), iter.getTopValue());
          iter.next();
        }
      }
     
      assertFalse(iter.hasTop());
    }
   
    reader.close();
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    }
   
    for (String arg : commandLine.getArgs()) {
      Path path = new Path(arg);
      CachableBlockFile.Reader rdr = new CachableBlockFile.Reader(fs, path, conf, null, null);
      Reader iter = new RFile.Reader(rdr);
     
      if (!arg.endsWith(".rf")) {
        throw new IllegalArgumentException("File must end with .rf");
      }
      String smallName = arg.substring(0, arg.length() - 3) + "_small.rf";
      String largeName = arg.substring(0, arg.length() - 3) + "_large.rf";
     
      int blockSize = (int) DefaultConfiguration.getDefaultConfiguration().getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE);
      Writer small = new RFile.Writer(new CachableBlockFile.Writer(fs, new Path(smallName), "gz", conf), blockSize);
      small.startDefaultLocalityGroup();
      Writer large = new RFile.Writer(new CachableBlockFile.Writer(fs, new Path(largeName), "gz", conf), blockSize);
      large.startDefaultLocalityGroup();

      iter.seek(new Range(), new ArrayList<ByteSequence>(), false);
      while (iter.hasTop()) {
        Key key = iter.getTopKey();
        Value value = iter.getTopValue();
        if (key.getSize() + value.getSize() < maxSize) {
          small.append(key, value);
        } else {
          large.append(key, value);
        }
        iter.next();
      }

      iter.close();
      large.close();
      small.close();
    }
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
    Path path = new Path(file);
    // long len = fs.getFileStatus(path).getLen();
    // FSDataInputStream in = fs.open(path);
    // Reader reader = new RFile.Reader(in, len , conf);
    CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, dataCache, indexCache);
    final Reader reader = new RFile.Reader(_cbr);
   
    return reader.getIndex();
  }
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.file.rfile.RFile.Reader

  • org.apache.accumulo.core.file.rfile.PrintInfo
  • org.apache.accumulo.core.file.rfile.RFileOperations
  • org.apache.accumulo.core.file.rfile.RFileTest
  • org.apache.accumulo.core.file.rfile.SplitLarge
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.