Package org.apache.blur.store.hdfs

Examples of org.apache.blur.store.hdfs.HdfsDirectory

86
87
88
89
90
91
92
93
94
95
96
      SortedSet<FileStatus> listStatus = sort(fileSystem.listStatus(path));
      List<HdfsDirectory> indexesToImport = new ArrayList<HdfsDirectory>();
      for (FileStatus fileStatus : listStatus) {
        Path file = fileStatus.getPath();
        if (fileStatus.isDir() && file.getName().endsWith(".commit")) {
          HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, file);
          if (!DirectoryReader.indexExists(hdfsDirectory)) {
            LOG.error("Directory found at [{0}] is not a vaild index.", file);
          } else {
            indexesToImport.add(hdfsDirectory);
          }
View Full Code Here
434
435
436
437
438
439
440
441
442
443
444
445
    Path tablePath = new Path(getTableDescriptor(table).tableUri);
    Path hdfsDirPath = new Path(tablePath, shard);

    BlurLockFactory lockFactory = new BlurLockFactory(_configuration, hdfsDirPath, _nodeName, BlurUtil.getPid());

    Directory directory = new HdfsDirectory(_configuration, hdfsDirPath);
    directory.setLockFactory(lockFactory);

    TableDescriptor descriptor = _clusterStatus.getTableDescriptor(true, _cluster, table);
    TableContext tableContext = TableContext.create(descriptor);
    ShardContext shardContext = ShardContext.create(tableContext, shard);
View Full Code Here
80
81
82
83
84
85
86
87
    ShardContext shardContext = new ShardContext();
    shardContext.tableContext = tableContext;
    shardContext.walShardPath = new Path(tableContext.getWalTablePath(), shard);
    shardContext.hdfsDirPath = new Path(tableContext.getTablePath(), shard);
    shardContext.shard = shard;
    shardContext.directory = new HdfsDirectory(tableContext.getConfiguration(), shardContext.hdfsDirPath);
    return shardContext;
  }
View Full Code Here
388
389
390
391
392
393
394
395
396
397
398
      _maxDocumentBufferSize = BlurOutputFormat.getMaxDocumentBufferSize(configuration);
      Path tableOutput = BlurOutputFormat.getOutputPath(configuration);
      String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, shardId);
      Path indexPath = new Path(tableOutput, shardName);
      _newIndex = new Path(indexPath, tmpDirName);
      _finalDir = new ProgressableDirectory(new HdfsDirectory(configuration, _newIndex),
          BlurOutputFormat.getProgressable());
      _finalDir.setLockFactory(NoLockFactory.getNoLockFactory());

      TableContext tableContext = TableContext.create(tableDescriptor);
      _fieldManager = tableContext.getFieldManager();
View Full Code Here
60
61
62
63
64
65
66
67
68
    file = new File(TMPDIR, "hdfsdirectorytest");
    rm(file);
    URI uri = new File(file, "hdfs").toURI();
    Path hdfsDirPath = new Path(uri.toString());
    Configuration conf = new Configuration();
    directory = new HdfsDirectory(conf, hdfsDirPath);
    seed = new Random().nextLong();
    random = new Random(seed);
  }
View Full Code Here
334
335
336
337
338
339
340
    return new BufferedDirectory(destDirectory, 32768);
  }

  protected Directory getDestDirectory(Configuration configuration, TableDescriptor descriptor, Path directoryPath)
      throws IOException {
    return new HdfsDirectory(configuration, directoryPath);
  }
View Full Code Here
131
132
133
134
135
136
137
138
139
    System.out.println("Counters: " + ctrs);

    Path path = new Path(tableUri, BlurUtil.getShardName(0));
    Collection<Path> commitedTasks = getCommitedTasks(path);
    assertEquals(1, commitedTasks.size());
    DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
    assertEquals(2, reader.numDocs());
    reader.close();
  }
View Full Code Here
185
186
187
188
189
190
191
192
193
    Path path = new Path(tableUri, BlurUtil.getShardName(0));
    Collection<Path> commitedTasks = getCommitedTasks(path);
    assertEquals(1, commitedTasks.size());

    DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
    assertEquals(80000, reader.numDocs());
    reader.close();
  }
View Full Code Here
228
229
230
231
232
233
234
235
236
237
238
    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
      Path path = new Path(tableUri, BlurUtil.getShardName(i));
      Collection<Path> commitedTasks = getCommitedTasks(path);
      assertEquals(1, commitedTasks.size());

      DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, commitedTasks.iterator().next()));
      total += reader.numDocs();
      reader.close();
    }
    assertEquals(80000, total);
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
    for (int i = 0; i < tableDescriptor.getShardCount(); i++) {
      Path path = new Path(tableUri, BlurUtil.getShardName(i));
      Collection<Path> commitedTasks = getCommitedTasks(path);
      assertTrue(multiple >= commitedTasks.size());
      for (Path p : commitedTasks) {
        DirectoryReader reader = DirectoryReader.open(new HdfsDirectory(conf, p));
        total += reader.numDocs();
        reader.close();
      }
    }
    assertEquals(80000, total);
View Full Code Here
TOP

Related Classes of org.apache.blur.store.hdfs.HdfsDirectory

  • com.yammer.metrics.core.Histogram
  • com.yammer.metrics.core.Meter
  • com.yammer.metrics.core.MetricName
  • org.apache.blur.manager.indexserver.DistributedIndexServer
  • org.apache.blur.manager.writer.IndexImporter
  • org.apache.blur.manager.writer.IndexImporterTest
  • org.apache.blur.mapreduce.BlurReducer
  • org.apache.blur.mapreduce.lib.BlurOutputFormat$BlurRecordWriter
  • org.apache.blur.mapreduce.lib.BlurOutputFormatTest
  • org.apache.blur.mapreduce.lib.GenericBlurRecordWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.