Examples of CountingIterator


Examples of net.myrrix.common.collection.CountingIterator

      }
    };

    log.info("Starting load test...");
    long start = System.currentTimeMillis();
    new Paralleler<Integer>(new CountingIterator(steps), processor, "Load").runInParallel();
    long end = System.currentTimeMillis();

    log.info("Finished {} steps in {}ms", steps, end - start);

    log.info("recommendedBecause: {}", recommendedBecause.getResult());
View Full Code Here

Examples of org.apache.accumulo.core.iterators.CountingIterator

    ArrayList<FileSKVIterator> readers = new ArrayList<FileSKVIterator>(filesToCompact.size());
    Span span = Trace.start("compact");
    try {
      long entriesCompacted = 0;
      List<SortedKeyValueIterator<Key,Value>> iters = openMapDataFiles(lgName, conf, fs, filesToCompact, extent, readers);
      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()));
      DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
     
      IteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
     
      SortedKeyValueIterator<Key,Value> itr = IteratorUtil.loadIterators(IteratorScope.majc, delIter, extent, acuTableConf, iterEnv);
     
      itr.seek(extent.toDataRange(), columnFamilies, inclusive);
     
      if (!inclusive) {
        mfw.startDefaultLocalityGroup();
      } else {
        mfw.startNewLocalityGroup(lgName, columnFamilies);
      }
     
      Span write = Trace.start("write");
      try {
        while (itr.hasTop() && isCompactionEnabled()) {
          mfw.append(itr.getTopKey(), itr.getTopValue());
          itr.next();
          entriesCompacted++;
        }
       
        if (itr.hasTop() && !isCompactionEnabled()) {
          // cancel major compaction operation
          try {
            try {
              mfw.close();
            } catch (IOException e) {
              log.error(e, e);
            }
            fs.delete(new Path(compactTmpName), true);
          } catch (Exception e) {
            log.warn("Failed to delete Canceled major compaction output file " + compactTmpName, e);
          }
          throw new MajorCompactionCanceledException();
        }
       
      } finally {
        MajorCompactionStats lgMajcStats = new MajorCompactionStats(citr.getCount(), entriesCompacted);
        majCStats.add(lgMajcStats);
        write.stop();
      }
     
    } finally {
View Full Code Here

Examples of org.apache.accumulo.core.iterators.CountingIterator

    ArrayList<FileSKVIterator> readers = new ArrayList<FileSKVIterator>(filesToCompact.size());
    Span span = Trace.start("compact");
    try {
      long entriesCompacted = 0;
      List<SortedKeyValueIterator<Key,Value>> iters = openMapDataFiles(lgName, conf, fs, filesToCompact, extent, readers);
      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()));
      DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
     
      IteratorEnvironment iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
     
      SortedKeyValueIterator<Key,Value> itr = IteratorUtil.loadIterators(IteratorScope.majc, delIter, extent, acuTableConf, iterEnv);
     
      itr.seek(extent.toDataRange(), columnFamilies, inclusive);
     
      if (!inclusive) {
        mfw.startDefaultLocalityGroup();
      } else {
        mfw.startNewLocalityGroup(lgName, columnFamilies);
      }
     
      Span write = Trace.start("write");
      try {
        while (itr.hasTop() && isCompactionEnabled()) {
          mfw.append(itr.getTopKey(), itr.getTopValue());
          itr.next();
          entriesCompacted++;
        }
       
        if (itr.hasTop() && !isCompactionEnabled()) {
          // cancel major compaction operation
          try {
            try {
              mfw.close();
            } catch (IOException e) {
              log.error(e, e);
            }
            fs.delete(new Path(compactTmpName), true);
          } catch (Exception e) {
            log.warn("Failed to delete Canceled major compaction output file " + compactTmpName, e);
          }
          throw new MajorCompactionCanceledException();
        }
       
      } finally {
        MajorCompactionStats lgMajcStats = new MajorCompactionStats(citr.getCount(), entriesCompacted);
        majCStats.add(lgMajcStats);
        write.stop();
      }
     
    } finally {
View Full Code Here

Examples of org.apache.accumulo.core.iterators.CountingIterator

          put(tm1, r, cf, cq, 6, r * cf * cq);
        }
      }
    }
   
    CountingIterator ci = new CountingIterator(new SortedMapIterator(tm1));
    ColumnFamilySkippingIterator cfi = new ColumnFamilySkippingIterator(ci);
    HashSet<ByteSequence> colfams = new HashSet<ByteSequence>();
    colfams.add(new ArrayByteSequence(String.format("%06d", 4)));
   
    Range range = new Range(nk(0, 4, 0, 6), true, nk(0, 400, 0, 6), true);
View Full Code Here

Examples of org.apache.accumulo.core.iterators.system.CountingIterator

     
      if (imm != null) {
        iters.add(imm.compactionIterator());
      }
     
      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()));
      DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
      ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
     

      // if(env.getIteratorScope() )
     
      TabletIteratorEnvironment iterEnv;
      if (env.getIteratorScope() == IteratorScope.majc)
        iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
      else if (env.getIteratorScope() == IteratorScope.minc)
        iterEnv = new TabletIteratorEnvironment(IteratorScope.minc, acuTableConf);
      else
        throw new IllegalArgumentException();
     
      SortedKeyValueIterator<Key,Value> itr = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(env.getIteratorScope(), cfsi, extent, acuTableConf,
          iterEnv));
     
      itr.seek(extent.toDataRange(), columnFamilies, inclusive);
     
      if (!inclusive) {
        mfw.startDefaultLocalityGroup();
      } else {
        mfw.startNewLocalityGroup(lgName, columnFamilies);
      }
     
      Span write = Trace.start("write");
      try {
        while (itr.hasTop() && env.isCompactionEnabled()) {
          mfw.append(itr.getTopKey(), itr.getTopValue());
          itr.next();
          entriesCompacted++;
        }
       
        if (itr.hasTop() && !env.isCompactionEnabled()) {
          // cancel major compaction operation
          try {
            try {
              mfw.close();
            } catch (IOException e) {
              log.error(e, e);
            }
            fs.delete(new Path(outputFile), true);
          } catch (Exception e) {
            log.warn("Failed to delete Canceled compaction output file " + outputFile, e);
          }
          throw new CompactionCanceledException();
        }
       
      } finally {
        CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
        majCStats.add(lgMajcStats);
        write.stop();
      }
     
    } finally {
View Full Code Here

Examples of org.apache.accumulo.core.iterators.system.CountingIterator

  @SuppressWarnings("unchecked")
    private static long process(TreeMap<Key,Value> sourceMap, TreeMap<Key,Value> resultMap, Range range, int numScans) throws IOException
    {
      org.apache.accumulo.core.iterators.SortedMapIterator source = new SortedMapIterator(sourceMap);
      CountingIterator counter = new CountingIterator(source);
      FirstEntryInRowIterator feiri = new FirstEntryInRowIterator();
      IteratorEnvironment env = new IteratorEnvironment(){

        public AccumuloConfiguration getConfig() {
          return null;
        }

        public IteratorScope getIteratorScope() {
          return null;
        }

        public boolean isFullMajorCompaction() {
          return false;
        }

        public void registerSideChannel(SortedKeyValueIterator<Key, Value> arg0) {

        }

        public SortedKeyValueIterator<Key, Value> reserveMapFileReader(
            String arg0) throws IOException {
          return null;
        }};

        feiri.init(counter, Collections.singletonMap(FirstEntryInRowIterator.NUM_SCANS_STRING_NAME, Integer.toString(numScans)), env);

        feiri.seek(range, Collections.EMPTY_SET, false);
        while(feiri.hasTop())
        {
          resultMap.put(feiri.getTopKey(), feiri.getTopValue());
          feiri.next();
        }
        return counter.getCount();
    }
View Full Code Here

Examples of org.apache.accumulo.core.iterators.system.CountingIterator

     
      if (imm != null) {
        iters.add(imm.compactionIterator());
      }
     
      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()));
      DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
      ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
     

      // if(env.getIteratorScope() )
     
      TabletIteratorEnvironment iterEnv;
      if (env.getIteratorScope() == IteratorScope.majc)
        iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
      else if (env.getIteratorScope() == IteratorScope.minc)
        iterEnv = new TabletIteratorEnvironment(IteratorScope.minc, acuTableConf);
      else
        throw new IllegalArgumentException();
     
      SortedKeyValueIterator<Key,Value> itr = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(env.getIteratorScope(), cfsi, extent, acuTableConf,
          iterEnv));
     
      itr.seek(extent.toDataRange(), columnFamilies, inclusive);
     
      if (!inclusive) {
        mfw.startDefaultLocalityGroup();
      } else {
        mfw.startNewLocalityGroup(lgName, columnFamilies);
      }
     
      Span write = Trace.start("write");
      try {
        while (itr.hasTop() && env.isCompactionEnabled()) {
          mfw.append(itr.getTopKey(), itr.getTopValue());
          itr.next();
          entriesCompacted++;
        }
       
        if (itr.hasTop() && !env.isCompactionEnabled()) {
          // cancel major compaction operation
          try {
            try {
              mfw.close();
            } catch (IOException e) {
              log.error(e, e);
            }
            fs.delete(new Path(outputFile), true);
          } catch (Exception e) {
            log.warn("Failed to delete Canceled compaction output file " + outputFile, e);
          }
          throw new CompactionCanceledException();
        }
       
      } finally {
        CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
        majCStats.add(lgMajcStats);
        write.stop();
      }
     
    } finally {
View Full Code Here

Examples of org.apache.accumulo.core.iterators.system.CountingIterator

          put(tm1, r, cf, cq, 6, r * cf * cq);
        }
      }
    }
   
    CountingIterator ci = new CountingIterator(new SortedMapIterator(tm1));
    ColumnFamilySkippingIterator cfi = new ColumnFamilySkippingIterator(ci);
    HashSet<ByteSequence> colfams = new HashSet<ByteSequence>();
    colfams.add(new ArrayByteSequence(String.format("%06d", 4)));
   
    Range range = new Range(nk(0, 4, 0, 6), true, nk(0, 400, 0, 6), true);
View Full Code Here

Examples of org.apache.accumulo.core.iterators.system.CountingIterator

     
      if (imm != null) {
        iters.add(imm.compactionIterator());
      }
     
      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()));
      DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
     
      // if(env.getIteratorScope() )
     
      TabletIteratorEnvironment iterEnv;
      if (env.getIteratorScope() == IteratorScope.majc)
        iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
      else if (env.getIteratorScope() == IteratorScope.minc)
        iterEnv = new TabletIteratorEnvironment(IteratorScope.minc, acuTableConf);
      else
        throw new IllegalArgumentException();
     
      SortedKeyValueIterator<Key,Value> itr = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(env.getIteratorScope(), delIter, extent, acuTableConf,
          iterEnv));
     
      itr.seek(extent.toDataRange(), columnFamilies, inclusive);
     
      if (!inclusive) {
        mfw.startDefaultLocalityGroup();
      } else {
        mfw.startNewLocalityGroup(lgName, columnFamilies);
      }
     
      Span write = Trace.start("write");
      try {
        while (itr.hasTop() && env.isCompactionEnabled()) {
          mfw.append(itr.getTopKey(), itr.getTopValue());
          itr.next();
          entriesCompacted++;
        }
       
        if (itr.hasTop() && !env.isCompactionEnabled()) {
          // cancel major compaction operation
          try {
            try {
              mfw.close();
            } catch (IOException e) {
              log.error(e, e);
            }
            fs.delete(new Path(outputFile), true);
          } catch (Exception e) {
            log.warn("Failed to delete Canceled compaction output file " + outputFile, e);
          }
          throw new CompactionCanceledException();
        }
       
      } finally {
        CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
        majCStats.add(lgMajcStats);
        write.stop();
      }
     
    } finally {
View Full Code Here

Examples of org.apache.accumulo.tserver.Compactor.CountingIterator

    tm.put(new Key("r1", "cf1", "cq1"), new Value("data1".getBytes()));
    tm.put(new Key("r2", "cf1", "cq1"), new Value("data2".getBytes()));

    SortedMapIterator smi = new SortedMapIterator(tm);

    CountingIterator ci = new CountingIterator(smi, new AtomicLong(0));
    CountingIterator dc1 = ci.deepCopy(null);
    CountingIterator dc2 = ci.deepCopy(null);

    readAll(ci);
    readAll(dc1);
    readAll(dc2);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.