Package org.apache.accumulo.server.tabletserver

Examples of org.apache.accumulo.server.tabletserver.NativeMap$NMSKVIter


   
    return f;
  }
 
  public void test11() {
    NativeMap nm = new NativeMap();
   
    // insert things with varying field sizes and value sizes
   
    // generate random data
    Random r = new Random(75);
   
    ArrayList<Pair<Key,Value>> testData = new ArrayList<Pair<Key,Value>>();
   
    for (int i = 0; i < 100000; i++) {
     
      Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), Math.abs(r.nextLong()), false, false);
      Value v = new Value(rlrf(r, 511));
     
      testData.add(new Pair<Key,Value>(k, v));
    }
   
    // insert unsorted data
    for (Pair<Key,Value> pair : testData) {
      nm.put(pair.getFirst(), pair.getSecond());
    }
   
    for (int i = 0; i < 2; i++) {
     
      // sort data
      Collections.sort(testData, new Comparator<Pair<Key,Value>>() {
        @Override
        public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
          return o1.getFirst().compareTo(o2.getFirst());
        }
      });
     
      // verify
      Iterator<Entry<Key,Value>> iter1 = nm.iterator();
      Iterator<Pair<Key,Value>> iter2 = testData.iterator();
     
      while (iter1.hasNext() && iter2.hasNext()) {
        Entry<Key,Value> e = iter1.next();
        Pair<Key,Value> p = iter2.next();
       
        if (!e.getKey().equals(p.getFirst()))
          throw new RuntimeException("Keys not equal");
       
        if (!e.getValue().equals(p.getSecond()))
          throw new RuntimeException("Values not equal");
      }
     
      if (iter1.hasNext())
        throw new RuntimeException("Not all of native map consumed");
     
      if (iter2.hasNext())
        throw new RuntimeException("Not all of test data consumed");
     
      System.out.println("test 11 nm mem " + nm.getMemoryUsed());
     
      // insert data again w/ different value
      Collections.shuffle(testData, r);
      // insert unsorted data
      for (Pair<Key,Value> pair : testData) {
        pair.getSecond().set(rlrf(r, 511));
        nm.put(pair.getFirst(), pair.getSecond());
      }
    }
   
    nm.delete();
  }
View Full Code Here


   
    nm.delete();
  }
 
  public void testBinary() {
    NativeMap nm = new NativeMap();
   
    byte emptyBytes[] = new byte[0];
   
    for (int i = 0; i < 256; i++) {
      for (int j = 0; j < 256; j++) {
        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
       
        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
        Value v = new Value(data);
       
        nm.put(k, v);
      }
    }
   
    Iterator<Entry<Key,Value>> iter = nm.iterator();
    for (int i = 0; i < 256; i++) {
      for (int j = 0; j < 256; j++) {
        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
       
        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
        Value v = new Value(data);
       
        assertTrue(iter.hasNext());
        Entry<Key,Value> entry = iter.next();
       
        assertEquals(k, entry.getKey());
        assertEquals(v, entry.getValue());
       
      }
    }
   
    assertFalse(iter.hasNext());
   
    for (int i = 0; i < 256; i++) {
      for (int j = 0; j < 256; j++) {
        byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
        byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
       
        Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
        Value v = new Value(data);
       
        Value v2 = nm.get(k);
       
        assertEquals(v, v2);
      }
    }
   
    nm.delete();
  }
View Full Code Here

   
    nm.delete();
  }
 
  public void testEmpty() {
    NativeMap nm = new NativeMap();
   
    assertTrue(nm.size() == 0);
    assertTrue(nm.getMemoryUsed() == 0);
   
    nm.delete();
  }
View Full Code Here

   
    nm.delete();
  }
 
  public void testConcurrentIter() throws IOException {
    NativeMap nm = new NativeMap();
   
    nm.put(nk(0), nv(0));
    nm.put(nk(1), nv(1));
    nm.put(nk(3), nv(3));
   
    SortedKeyValueIterator<Key,Value> iter = nm.skvIterator();
   
    // modify map after iter created
    nm.put(nk(2), nv(2));
   
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), nk(0));
    iter.next();
   
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), nk(1));
    iter.next();
   
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), nk(2));
    iter.next();
   
    assertTrue(iter.hasTop());
    assertEquals(iter.getTopKey(), nk(3));
    iter.next();
   
    assertFalse(iter.hasTop());
   
    nm.delete();
  }
View Full Code Here

  }
 
  static void runPerformanceTest(int numRows, int numCols, int numLookups, String mapType) {
   
    SortedMap<Key,Value> tm = null;
    NativeMap nm = null;
   
    if (mapType.equals("SKIP_LIST"))
      tm = new ConcurrentSkipListMap<Key,Value>();
    else if (mapType.equals("TREE_MAP"))
      tm = Collections.synchronizedSortedMap(new TreeMap<Key,Value>());
    else if (mapType.equals("NATIVE_MAP"))
      nm = new NativeMap();
    else
      throw new IllegalArgumentException(" map type must be SKIP_LIST, TREE_MAP, or NATIVE_MAP");
   
    Random rand = new Random(19);
   
    // puts
    long tps = System.currentTimeMillis();
   
    if (nm != null) {
      for (int i = 0; i < numRows; i++) {
        int row = rand.nextInt(1000000000);
        Mutation m = nm(row);
        for (int j = 0; j < numCols; j++) {
          int col = rand.nextInt(1000000);
          Value val = new Value("test".getBytes(Constants.UTF8));
          pc(m, col, val);
        }
        nm.mutate(m, i);
      }
    } else {
      for (int i = 0; i < numRows; i++) {
        int row = rand.nextInt(1000000000);
        for (int j = 0; j < numCols; j++) {
          int col = rand.nextInt(1000000);
          Key key = nk(row, col);
          Value val = new Value("test".getBytes(Constants.UTF8));
          tm.put(key, val);
        }
      }
    }
   
    long tpe = System.currentTimeMillis();
   
    // Iteration
    Iterator<Entry<Key,Value>> iter;
    if (nm != null) {
      iter = nm.iterator();
    } else {
      iter = tm.entrySet().iterator();
    }
   
    long tis = System.currentTimeMillis();
   
    while (iter.hasNext()) {
      iter.next();
    }
   
    long tie = System.currentTimeMillis();
   
    rand = new Random(19);
    int rowsToLookup[] = new int[numLookups];
    int colsToLookup[] = new int[numLookups];
    for (int i = 0; i < Math.min(numLookups, numRows); i++) {
      int row = rand.nextInt(1000000000);
      int col = -1;
      for (int j = 0; j < numCols; j++) {
        col = rand.nextInt(1000000);
      }
     
      rowsToLookup[i] = row;
      colsToLookup[i] = col;
    }
   
    // get
   
    long tgs = System.currentTimeMillis();
    if (nm != null) {
      for (int i = 0; i < numLookups; i++) {
        Key key = nk(rowsToLookup[i], colsToLookup[i]);
        if (nm.get(key) == null) {
          throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
        }
      }
    } else {
      for (int i = 0; i < numLookups; i++) {
        Key key = nk(rowsToLookup[i], colsToLookup[i]);
        if (tm.get(key) == null) {
          throw new RuntimeException("Did not find " + rowsToLookup[i] + " " + colsToLookup[i] + " " + i);
        }
      }
    }
    long tge = System.currentTimeMillis();
   
    long memUsed = 0;
    if (nm != null) {
      memUsed = nm.getMemoryUsed();
    }
   
    int size = (nm == null ? tm.size() : nm.size());
   
    // delete
    long tds = System.currentTimeMillis();
   
    if (nm != null)
      nm.delete();
   
    long tde = System.currentTimeMillis();
   
    if (tm != null)
      tm.clear();
View Full Code Here

   
    for (int tCount = 0; tCount < numThreads; tCount++) {
      Runnable r = new Runnable() {
        @Override
        public void run() {
          NativeMap nm = new NativeMap();
         
          Random r = new Random();
         
          OpTimer opTimer = new OpTimer(log, Level.INFO);
         
          opTimer.start("Creating map of size " + mapSizePerThread);
         
          for (int i = 0; i < mapSizePerThread; i++) {
            String row = String.format("r%08d", i);
            String val = row + "v";
            put(nm, row, val, i);
          }
         
          opTimer.stop("Created map of size " + nm.size() + " in %DURATION%");
         
          opTimer.start("Doing " + getsPerThread + " gets()");
         
          for (int i = 0; i < getsPerThread; i++) {
            String row = String.format("r%08d", r.nextInt(mapSizePerThread));
            String val = row + "v";
           
            Value value = nm.get(new Key(new Text(row)));
            if (value == null || !value.toString().equals(val)) {
              log.error("nm.get(" + row + ") failed");
            }
          }
         
          opTimer.stop("Finished " + getsPerThread + " gets in %DURATION%");
         
          int scanned = 0;
         
          opTimer.start("Doing " + getsPerThread + " random iterations");
         
          for (int i = 0; i < getsPerThread; i++) {
            int startRow = r.nextInt(mapSizePerThread);
            String row = String.format("r%08d", startRow);
           
            Iterator<Entry<Key,Value>> iter = nm.iterator(new Key(new Text(row)));
           
            int count = 0;
           
            while (iter.hasNext() && count < 10) {
              String row2 = String.format("r%08d", startRow + count);
              String val2 = row2 + "v";
             
              Entry<Key,Value> entry = iter.next();
              if (!entry.getValue().toString().equals(val2) || !entry.getKey().equals(new Key(new Text(row2)))) {
                log.error("nm.iter(" + row2 + ") failed row = " + row + " count = " + count + " row2 = " + row + " val2 = " + val2);
              }
             
              count++;
            }
           
            scanned += count;
          }
         
          opTimer.stop("Finished " + getsPerThread + " random iterations (scanned = " + scanned + ") in %DURATION%");
         
          nm.delete();
        }
      };
     
      Thread t = new Thread(r);
      t.start();
View Full Code Here

          int inserts = 0;
          int removes = 0;
         
          for (int i = 0; i < mapsPerThread; i++) {
           
            NativeMap nm = new NativeMap();
           
            for (int j = 0; j < insertsPerMapPerThread; j++) {
              String row = String.format("r%08d", j % rowRange);
              String val = row + "v";
              put(nm, row, val, j);
              inserts++;
            }
           
            if (doRemoves) {
              Iterator<Entry<Key,Value>> iter = nm.iterator();
              while (iter.hasNext()) {
                iter.next();
                iter.remove();
                removes++;
              }
            }
           
            nm.delete();
          }
         
          System.out.println("inserts " + inserts + " removes " + removes + " " + Thread.currentThread().getName());
        }
      };
View Full Code Here

          int inserts = 0;
         
          for (int i = 0; i < insertsPerThread / 100.0; i++) {
            int map = r.nextInt(numMaps);
           
            NativeMap nm;
           
            synchronized (nativeMaps) {
              nm = nativeMaps.get(map);
              if (nm == null) {
                nm = new NativeMap();
                nativeMaps.put(map, nm);
               
              }
            }
           
View Full Code Here

TOP

Related Classes of org.apache.accumulo.server.tabletserver.NativeMap$NMSKVIter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.