Package proj.zoie.api

Examples of proj.zoie.api.DefaultDirectoryManager


    File idxDir=getIdxDir();
    DefaultZoieVersionFactory defaultZoieVersionFactory = new DefaultZoieVersionFactory();
    final ZoieSystem<IndexReader,String,DefaultZoieVersion> idxSystem=createZoie(idxDir,true,defaultZoieVersionFactory);
    idxSystem.start();

    DirectoryManager dirMgr = new DefaultDirectoryManager(idxDir,defaultZoieVersionFactory);

    String query="zoie";
    QueryParser parser=new QueryParser(Version.LUCENE_CURRENT,"contents",idxSystem.getAnalyzer());
    Query q=null;
    try
    {
      q=parser.parse(query);
    }
    catch (ParseException e)
    {
      throw new ZoieException(e.getMessage(),e);
    }

   
    try
    {
      List<DataEvent<String,DefaultZoieVersion>> list;
      DefaultZoieVersion zvt = null;

      list=new ArrayList<DataEvent<String,DefaultZoieVersion>>(TestData.testdata.length);
      for (int i=0;i<TestData.testdata.length;++i)
      {
        zvt = new DefaultZoieVersion();       
        zvt.setVersionId(i);
        list.add(new DataEvent<String,DefaultZoieVersion>(TestData.testdata[i],zvt));
      }
      idxSystem.consume(list);
      idxSystem.flushEvents(100000);
     
      DefaultZoieVersion versionExported = new DefaultZoieVersion();
      versionExported.setVersionId(zvt.getVersionId());
         
      assertEquals("index version mismatch after first flush", TestData.testdata.length - 1, zvt.getVersionId());
     
     
      int hits = countHits(idxSystem, q);

      RandomAccessFile exportFile;
      FileChannel channel;

      exportFile = new RandomAccessFile(new File(getTmpDir(), "zoie_export.dat"), "rw");
      channel = exportFile.getChannel();
      idxSystem.exportSnapshot(channel);
      channel.close();
      exportFile.close();
      exportFile = null;
      channel = null;
    
      list=new ArrayList<DataEvent<String,DefaultZoieVersion>>(TestData.testdata2.length);
      for (int i=0; i < TestData.testdata2.length; ++i)
      {
        zvt = new DefaultZoieVersion();
        zvt.setVersionId(TestData.testdata.length+i);
       
        list.add(new DataEvent<String, DefaultZoieVersion>(TestData.testdata.length + TestData.testdata2[i],zvt));
      }
      idxSystem.consume(list);
      idxSystem.flushEvents(100000);
      zvt = (DefaultZoieVersion)dirMgr.getVersion();
      assertEquals("index version mismatch after second flush", TestData.testdata.length + TestData.testdata2.length - 1, zvt.getVersionId());
      assertEquals("should have no hits", 0, countHits(idxSystem, q));
     
      exportFile = new RandomAccessFile(new File(getTmpDir(), "zoie_export.dat"), "r");
      channel = exportFile.getChannel();
      idxSystem.importSnapshot(channel);
      idxSystem.flushEvents(10000);
      channel.close();
      exportFile.close();

      assertEquals("count is wrong", hits, countHits(idxSystem, q));

     
      zvt = (DefaultZoieVersion)dirMgr.getVersion();     
      assertEquals("imported version is wrong", versionExported.getVersionId(), zvt.getVersionId());    }
    
    catch(ZoieException e)
    {
      throw e;
View Full Code Here


    int numReplicas = Integer.parseInt(args[2]);
    System.out.println("source index: "+srcIndex.getAbsolutePath());
    System.out.println("target index: "+targetIndex.getAbsolutePath());
    System.out.println("num replications: "+numReplicas);
    IndexReader reader = null;
    DirectoryManager srcDirMgr = new DefaultDirectoryManager(srcIndex, new DefaultZoieVersionFactory());
    try
    {
      Directory dir = srcDirMgr.getDirectory();
      reader=IndexReader.open(dir,true);
      System.out.println("source index, numdocs: "+reader.numDocs());
    }
    catch(IOException e)
    {
      e.printStackTrace();
    }
    finally
    {
      if (reader!=null)
      {
        reader.close();
      }
      reader = null;
    }
    DirectoryManager targetDirMgr = new DefaultDirectoryManager(targetIndex, new DefaultZoieVersionFactory());

    Directory targetDir = targetDirMgr.getDirectory(true);
   
    IndexWriter writer = null;
    try
    {
      writer = new IndexWriter(targetDir,new StandardAnalyzer(Version.LUCENE_CURRENT),true,MaxFieldLength.UNLIMITED);
View Full Code Here

    {
      partDir.mkdirs();
      log.info("nodeId="+nodeId+", partition=" + partitionId + " does not exist, directory created.");
    }
   
    DirectoryManager dirMgr = new DefaultDirectoryManager(partDir, _dirMode);

    ZoieSystem<BoboIndexReader,T> zoie = new ZoieSystem<BoboIndexReader,T>(dirMgr, _interpreter, _indexReaderDecorator, _zoieConfig);
    if (_optimizeScheduler != null) {
      zoie.setOptimizeScheduler(_optimizeScheduler);
    }
View Full Code Here

      File[] files = _zoieOneRoot.listFiles();
      Arrays.sort(files);
      for (int i = files.length - 1; i >= 0; --i) {
        if (new File(files[i], COMMIT_FILE).exists()) {
          log.info("Found latest zoieOne index: " + files[i].getAbsolutePath());
          DirectoryManager dirMgr = new DefaultDirectoryManager(files[i], dirMode);
          _zoieOne = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zoieConfig);
          break;
        }
      }
    }
View Full Code Here

      log.error(e.getMessage(), e);
      return false;
    }

    // offline line index is not moving, so MMAP should be the most efficient.
    DirectoryManager dirMgr = new DefaultDirectoryManager(dest, DIRECTORY_MODE.MMAP);
    Zoie<R, D> zoie = new ZoieSystem(dirMgr, _interpreter, _decorator, _zoieConfig);
    if (_running) zoie.start();

    if (_zoieOne != null) {
      final Zoie toBeShutdown = _zoieOne;
View Full Code Here

    File idxDir = getIdxDir();
    final ZoieSystem<IndexReader, String> idxSystem = createZoie(idxDir, true,
      ZoieConfig.DEFAULT_VERSION_COMPARATOR);
    idxSystem.start();

    DirectoryManager dirMgr = new DefaultDirectoryManager(idxDir);

    String query = "zoie";
    QueryParser parser = new QueryParser(Version.LUCENE_43, "contents", idxSystem.getAnalyzer());
    Query q = null;
    try {
      q = parser.parse(query);
    } catch (Exception e) {
      throw new ZoieException(e.getMessage(), e);
    }

    try {
      List<DataEvent<String>> list;

      list = new ArrayList<DataEvent<String>>(DataForTests.testdata.length);
      for (int i = 0; i < DataForTests.testdata.length; ++i) {
        list.add(new DataEvent<String>(DataForTests.testdata[i], "" + i));
      }
      idxSystem.consume(list);
      idxSystem.flushEvents(100000);

      assertEquals("index version mismatch after first flush", DataForTests.testdata.length - 1,
        DataForTests.testdata.length - 1);

      int hits = countHits(idxSystem, q);

      RandomAccessFile exportFile;
      FileChannel channel;

      exportFile = new RandomAccessFile(new File(getTmpDir(), "zoie_export.dat"), "rw");
      channel = exportFile.getChannel();
      idxSystem.exportSnapshot(channel);
      channel.close();
      exportFile.close();
      exportFile = null;
      channel = null;

      list = new ArrayList<DataEvent<String>>(DataForTests.testdata2.length);
      for (int i = 0; i < DataForTests.testdata2.length; ++i) {

        list.add(new DataEvent<String>(DataForTests.testdata.length + DataForTests.testdata2[i], ""
            + (DataForTests.testdata.length + i)));
      }
      idxSystem.consume(list);
      idxSystem.flushEvents(100000);
      String zvt = dirMgr.getVersion();
      assertEquals("index version mismatch after second flush", DataForTests.testdata.length
          + DataForTests.testdata2.length - 1, (long) Long.valueOf(zvt));
      assertEquals("should have no hits", 0, countHits(idxSystem, q));

      exportFile = new RandomAccessFile(new File(getTmpDir(), "zoie_export.dat"), "r");
View Full Code Here

    try {
      log.info("rolling forward with new path: " + _location.getCanonicalPath());
    } catch (IOException e) {
      log.error(e);
    }
    _currentDirMgr = new DefaultDirectoryManager(_location, _mode);
    isRecentlyChanged = true;
    setNextUpdateTime();
    return isRecentlyChanged;
  }
View Full Code Here

    List<ZoieSystem<R, D>> archives = new ArrayList<ZoieSystem<R, D>>();
    long t0 = System.currentTimeMillis();
    List<File> dirs = _dirMgrFactory.getAllArchivedDirs();
    for (File dir : dirs) {
      try {
        DirectoryManager dirMgr = new DefaultDirectoryManager(dir, _dirMgrFactory.getMode());
        ZoieSystem<R, D> zoie = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zConfig);
        zoie.start();
        archives.add(zoie);
      } catch (Exception e) {
        log.error("Load index: " + dir + " failed.", e);
View Full Code Here

  @Deprecated
  public ZoieSystem(File idxDir, ZoieIndexableInterpreter<D> interpreter,
      IndexReaderDecorator<R> indexReaderDecorator, Analyzer analyzer, Similarity similarity,
      int batchSize, long batchDelay, boolean rtIndexing, Comparator<String> versionComparator,
      boolean skipBadRecord) {
    this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, analyzer,
        similarity, batchSize, batchDelay, rtIndexing, versionComparator, skipBadRecord);
  }
View Full Code Here

   *          configuration object
   */
  @SuppressWarnings({ "deprecation", "unchecked" })
  public ZoieSystem(File idxDir, ZoieIndexableInterpreter<D> interpreter,
      IndexReaderDecorator<R> indexReaderDecorator, ZoieConfig zoieConfig) {
    this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, zoieConfig
        .getDocidMapperFactory(), zoieConfig.getAnalyzer(), zoieConfig.getSimilarity(), zoieConfig
        .getBatchSize(), zoieConfig.getBatchDelay(), zoieConfig.isRtIndexing(), zoieConfig
        .getMaxBatchSize(), zoieConfig.getReadercachefactory(), (RAMIndexFactory<R>) zoieConfig
        .getRamIndexFactory(), zoieConfig.getVersionComparator(), zoieConfig.isSkipBadRecord());
    readercache.setFreshness(zoieConfig.getFreshness());
View Full Code Here

TOP

Related Classes of proj.zoie.api.DefaultDirectoryManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.