Examples of org.apache.hadoop.contrib.index.lucene.FileSystemDirectory

Package org.apache.hadoop.contrib.index.lucene

Examples of org.apache.hadoop.contrib.index.lucene.FileSystemDirectory

org.apache.hadoop.contrib.index.lucene.FileSystemDirectory
This class implements a Lucene Directory on top of a general FileSystem. Currently it does not support locking.


    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);
    IndexSearcher searcher = new IndexSearcher(reader);
    Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));


    assertEquals(numRuns * numDocsPerRun, hits.length());


    int[] counts = new int[numDocsPerRun];
    for (int i = 0; i < hits.length(); i++) {
      Document doc = hits.doc(i);
      counts[Integer.parseInt(doc.get("id"))]++;
    }


    for (int i = 0; i < numDocsPerRun; i++) {
      assertEquals(numRuns, counts[i]);
    }


    // max field length is 2, so "dot" is also indexed but not "org"
    hits = searcher.search(new TermQuery(new Term("content", "dot")));
    assertEquals(numRuns, hits.length());


    hits = searcher.search(new TermQuery(new Term("content", "org")));
    assertEquals(0, hits.length());


    searcher.close();
    reader.close();


    // open and close an index writer with KeepOnlyLastCommitDeletionPolicy
    // to remove earlier checkpoints
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      IndexWriter writer =
          new IndexWriter(dir, false, null,
              new KeepOnlyLastCommitDeletionPolicy());
      writer.close();

View Full Code Here


    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);
    IndexSearcher searcher = new IndexSearcher(reader);
    Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));


    assertEquals(numRuns * numDocsPerRun, hits.length());


    int[] counts = new int[numDocsPerRun];
    for (int i = 0; i < hits.length(); i++) {
      Document doc = hits.doc(i);
      counts[Integer.parseInt(doc.get("id"))]++;
    }


    for (int i = 0; i < numDocsPerRun; i++) {
      assertEquals(numRuns, counts[i]);
    }


    // max field length is 2, so "dot" is also indexed but not "org"
    hits = searcher.search(new TermQuery(new Term("content", "dot")));
    assertEquals(numRuns, hits.length());


    hits = searcher.search(new TermQuery(new Term("content", "org")));
    assertEquals(0, hits.length());


    searcher.close();
    reader.close();


    // open and close an index writer with KeepOnlyLastCommitDeletionPolicy
    // to remove earlier checkpoints
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      IndexWriter writer =
          new IndexWriter(dir, false, null,
              new KeepOnlyLastCommitDeletionPolicy());
      writer.close();

View Full Code Here

  private void verify(Shard[] shards) throws IOException {
    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);

View Full Code Here

    for (int i = 0; i < shards.length; i++) {
      Path path = new Path(shards[i].getDirectory());
      long generation = -1;


      if (fs.exists(path)) {
        FileSystemDirectory dir = null;


        try {
          dir = new FileSystemDirectory(fs, path, false, conf);
          generation = LuceneUtil.getCurrentSegmentGeneration(dir);
        } finally {
          if (dir != null) {
            dir.close();
          }
        }
      }


      if (generation != shards[i].getGeneration()) {

View Full Code Here


    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);
    IndexSearcher searcher = new IndexSearcher(reader);
    Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));


    assertEquals(numRuns * numDocsPerRun, hits.length());


    int[] counts = new int[numDocsPerRun];
    for (int i = 0; i < hits.length(); i++) {
      Document doc = hits.doc(i);
      counts[Integer.parseInt(doc.get("id"))]++;
    }


    for (int i = 0; i < numDocsPerRun; i++) {
      assertEquals(numRuns, counts[i]);
    }


    // max field length is 2, so "dot" is also indexed but not "org"
    hits = searcher.search(new TermQuery(new Term("content", "dot")));
    assertEquals(numRuns, hits.length());


    hits = searcher.search(new TermQuery(new Term("content", "org")));
    assertEquals(0, hits.length());


    searcher.close();
    reader.close();


    // open and close an index writer with KeepOnlyLastCommitDeletionPolicy
    // to remove earlier checkpoints
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      IndexWriter writer =
          new IndexWriter(dir, false, null,
              new KeepOnlyLastCommitDeletionPolicy());
      writer.close();

View Full Code Here

  private void verify(Shard[] shards) throws IOException {
    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);

View Full Code Here

    for (int i = 0; i < shards.length; i++) {
      Path path = new Path(shards[i].getDirectory());
      long generation = -1;


      if (fs.exists(path)) {
        FileSystemDirectory dir = null;


        try {
          dir = new FileSystemDirectory(fs, path, false, conf);
          generation = LuceneUtil.getCurrentSegmentGeneration(dir);
        } finally {
          if (dir != null) {
            dir.close();
          }
        }
      }


      if (generation != shards[i].getGeneration()) {

View Full Code Here


    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);
    IndexSearcher searcher = new IndexSearcher(reader);
    Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));


    assertEquals(numRuns * numDocsPerRun, hits.length());


    int[] counts = new int[numDocsPerRun];
    for (int i = 0; i < hits.length(); i++) {
      Document doc = hits.doc(i);
      counts[Integer.parseInt(doc.get("id"))]++;
    }


    for (int i = 0; i < numDocsPerRun; i++) {
      assertEquals(numRuns, counts[i]);
    }


    // max field length is 2, so "dot" is also indexed but not "org"
    hits = searcher.search(new TermQuery(new Term("content", "dot")));
    assertEquals(numRuns, hits.length());


    hits = searcher.search(new TermQuery(new Term("content", "org")));
    assertEquals(0, hits.length());


    searcher.close();
    reader.close();


    // open and close an index writer with KeepOnlyLastCommitDeletionPolicy
    // to remove earlier checkpoints
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      IndexWriter writer =
          new IndexWriter(dir, false, null,
              new KeepOnlyLastCommitDeletionPolicy());
      writer.close();

View Full Code Here

  private void verify(Shard[] shards) throws IOException {
    // verify the index
    IndexReader[] readers = new IndexReader[shards.length];
    for (int i = 0; i < shards.length; i++) {
      Directory dir =
          new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
              false, conf);
      readers[i] = IndexReader.open(dir);
    }


    IndexReader reader = new MultiReader(readers);

View Full Code Here

TOP

Related Classes of org.apache.hadoop.contrib.index.lucene.FileSystemDirectory

org.apache.hadoop.contrib.index.mapred.IndexUpdater

org.apache.hadoop.contrib.index.mapred.TestDistributionPolicy

org.apache.hadoop.contrib.index.mapred.TestIndexUpdater

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.Path

org.apache.lucene.store.Lock

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.