Package org.apache.lucene.index

Examples of org.apache.lucene.index.IndexCommit


          name, _tableContext.getTable(), _shardContext.getShard());
      throw new IOException("A Snapshot already exists with the same name [" + name + "] on " +
        "["+_tableContext.getTable()+"/"+_shardContext.getShard()+"].");
    }
    _writer.commit();
    IndexCommit indexCommit = snapshotter.snapshot(name);
   
    /*
     * Persist the snapshots info into a tmp file under the snapshots sub-folder
     * and once writing is finished, close the writer.
     * Now rename the tmp file to an actual snapshots file.
     * This make the file write an atomic operation
     *
     * The name of the file is the snapshot name and its contents specify the segments file name
     */
    String segmentsFilename = indexCommit.getSegmentsFileName();
    FileSystem fileSystem = getFileSystem();
    Path shardSnapshotsDirPath = getSnapshotsDirectoryPath();
    BlurUtil.createPath(fileSystem, shardSnapshotsDirPath);
    Path newTmpSnapshotFile = new Path(shardSnapshotsDirPath, name + SNAPSHOTS_TMPFILE_EXTENSION);
    BufferedWriter br=new BufferedWriter(new OutputStreamWriter(fileSystem.create(newTmpSnapshotFile,true)));
View Full Code Here


      long latestGeneration = (Long) response.get(GENERATION);
      if (latestVersion == 0L) {
        //there is nothing to be replicated
        return false;
      }
      IndexCommit commit;
      RefCounted<SolrIndexSearcher> searcherRefCounted = null;
      try {
        searcherRefCounted = core.getNewestSearcher(false);
        commit = searcherRefCounted.get().getReader().getIndexCommit();
      } finally {
        if (searcherRefCounted != null)
          searcherRefCounted.decref();
      }
      if (commit.getVersion() == latestVersion && commit.getGeneration() == latestGeneration) {
        //master and slave are alsready in sync just return
        LOG.info("Slave in sync with master.");
        return false;
      }
      LOG.info("Master's version: " + latestVersion + ", generation: " + latestGeneration);
      LOG.info("Slave's version: " + commit.getVersion() + ", generation: " + commit.getGeneration());
      LOG.info("Starting replication process");
      // get the list of files first
      fetchFileList(latestVersion);
      // this can happen if the commit point is deleted before we fetch the file list.
      if(filesToDownload.isEmpty()) return false;
      LOG.info("Number of files in latest index in master: " + filesToDownload.size());

      // Create the sync service
      fsyncService = Executors.newSingleThreadExecutor();
      // use a synchronized list because the list is read by other threads (to show details)
      filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
      // if the generateion of master is older than that of the slave , it means they are not compatible to be copied
      // then a new index direcory to be created and all the files need to be copied
      boolean isFullCopyNeeded = commit.getGeneration() >= latestGeneration;
      File tmpIndexDir = createTempindexDir(core);
      if (isIndexStale())
        isFullCopyNeeded = true;
      successfulInstall = false;
      boolean deleteTmpIdxDir = true;
View Full Code Here

      return;
    }
    // This command does not give the current index version of the master
    // It gives the current 'replicateable' index version
   if (command.equals(CMD_INDEX_VERSION)) {
      IndexCommit commitPoint = indexCommitPoint;  // make a copy so it won't change
      if (commitPoint != null && replicationEnabled.get()) {
        //
        // There is a race condition here.  The commit point may be changed / deleted by the time
        // we get around to reserving it.  This is a very small window though, and should not result
        // in a catastrophic failure, but will result in the client getting an empty file list for
        // the CMD_GET_FILE_LIST command.
        //
        core.getDeletionPolicy().setReserveDuration(commitPoint.getVersion(), reserveCommitDuration);
        rsp.add(CMD_INDEX_VERSION, commitPoint.getVersion());
        rsp.add(GENERATION, commitPoint.getGeneration());
      } else {
        // This happens when replication is not configured to happen after startup and no commit/optimize
        // has happened yet.
        rsp.add(CMD_INDEX_VERSION, 0L);
        rsp.add(GENERATION, 0L);
View Full Code Here

  }

  private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) {
    try {
      IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
      IndexCommit indexCommit = delPolicy.getLatestCommit();

      if(indexCommit == null) {
        indexCommit = req.getSearcher().getReader().getIndexCommit();
      }
View Full Code Here

    if (v == null) {
      rsp.add("status", "no indexversion specified");
      return;
    }
    long version = Long.parseLong(v);
    IndexCommit commit = core.getDeletionPolicy().getCommitPoint(version);
    if (commit == null) {
      rsp.add("status", "invalid indexversion");
      return;
    }
    // reserve the indexcommit for sometime
    core.getDeletionPolicy().setReserveDuration(version, reserveCommitDuration);
    List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
    try {
      //get all the files in the commit
      //use a set to workaround possible Lucene bug which returns same file name multiple times
      Collection<String> files = new HashSet<String>(commit.getFileNames());
      for (String fileName : files) {
        if(fileName.endsWith(".lock")) continue;
        File file = new File(core.getIndexDir(), fileName);
        Map<String, Object> fileMeta = getFileInfo(file);
        result.add(fileMeta);
View Full Code Here

    details.add("isSlave", String.valueOf(isSlave));
    long[] versionAndGeneration = getIndexVersion();
    details.add("indexVersion", versionAndGeneration[0]);
    details.add(GENERATION, versionAndGeneration[1]);

    IndexCommit commit = indexCommitPoint;  // make a copy so it won't change

    if (isMaster) {
      if (includeConfFiles != null) master.add(CONF_FILES, includeConfFiles);
      master.add(REPLICATE_AFTER, getReplicateAfterStrings());
      master.add("replicationEnabled", String.valueOf(replicationEnabled.get()));
    }

    if (isMaster && commit != null) {
      master.add("replicatableIndexVersion", commit.getVersion());
      master.add("replicatableGeneration", commit.getGeneration());
    }

    SnapPuller snapPuller = tempSnapPuller;
    if (showSlaveDetails && snapPuller != null) {
      Properties props = loadReplicationProperties();
View Full Code Here

    return props;
  }


  void refreshCommitpoint() {
    IndexCommit commitPoint = core.getDeletionPolicy().getLatestCommit();
    if(replicateOnCommit || (replicateOnOptimize && commitPoint.isOptimized())) {
      indexCommitPoint = commitPoint;
    }
  }
View Full Code Here

      /**
       * This refreshes the latest replicateable index commit and optionally can create Snapshots as well
       */
      public void postCommit() {
        IndexCommit currentCommitPoint = core.getDeletionPolicy().getLatestCommit();

        if (getCommit) {
          // IndexCommit oldCommitPoint = indexCommitPoint;
          indexCommitPoint = currentCommitPoint;

View Full Code Here

    // this will happen rarely, so just synchronize everything
    // for safety and to avoid race conditions

    synchronized (this) {
      long maxCommitAgeTimeStamp = -1L;
      IndexCommit newest = commits.get(commits.size() - 1);
      log.info("newest commit = " + newest.getVersion());

      int optimizedKept = newest.isOptimized() ? 1 : 0;
      int totalKept = 1;

      // work our way from newest to oldest, skipping the first since we always want to keep it.
      for (int i=commits.size()-2; i>=0; i--) {
        IndexCommit commit = commits.get(i);

        // delete anything too old, regardless of other policies
        try {
          if (maxCommitAge != null) {
            if (maxCommitAgeTimeStamp==-1) {
              DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.US);
              maxCommitAgeTimeStamp = dmp.parseMath(maxCommitAge).getTime();
            }
            if (commit.getTimestamp() < maxCommitAgeTimeStamp) {
              commit.delete();
              continue;
            }
          }
        } catch (Exception e) {
          log.warn("Exception while checking commit point's age for deletion", e);
        }

        if (optimizedKept < maxOptimizedCommitsToKeep && commit.isOptimized()) {
          totalKept++;
          optimizedKept++;
          continue;
        }

        if (totalKept < maxCommitsToKeep) {
          totalKept++;
          continue;
        }
                                                 
        commit.delete();
      }

    } // end synchronized
  }
View Full Code Here

  public void testKeepOptimizedOnlyCommits() {

    IndexDeletionPolicyWrapper delPolicy = h.getCore().getDeletionPolicy();
    addDocs();
    Map<Long, IndexCommit> commits = delPolicy.getCommits();
    IndexCommit latest = delPolicy.getLatestCommit();
    for (Long version : commits.keySet()) {
      if (commits.get(version) == latest)
        continue;
      assertTrue(commits.get(version).isOptimized());
    }
View Full Code Here

TOP

Related Classes of org.apache.lucene.index.IndexCommit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.