Package tachyon

Examples of tachyon.UnderFileSystem


    Assert.assertEquals(ByteBuffer.wrap(buf), dataBuf);
  }

  private void initializeStorageDir(StorageDir dir, long userId) throws IOException {
    dir.initailize();
    UnderFileSystem ufs = dir.getUfs();
    ufs.mkdirs(dir.getUserTempPath(userId), true);
    CommonUtils.changeLocalFileToFullPermission(dir.getUserTempPath(userId));
  }
View Full Code Here


  private void setupStreamFromUnderFs(long offset, Object conf) throws IOException {
    String checkpointPath = mFile.getUfsPath();
    if (!checkpointPath.equals("")) {
      LOG.info("May stream from underlayer fs: " + checkpointPath);
      UnderFileSystem underfsClient = UnderFileSystem.get(checkpointPath, conf);
      try {
        mCheckpointInputStream = underfsClient.open(checkpointPath);
        while (offset > 0) {
          long skipped = mCheckpointInputStream.skip(offset);
          offset -= skipped;
          if (skipped == 0) {
            throw new IOException("Failed to find the start position " + offset + " for block "
View Full Code Here

      // throw new IOException("Failed to create folder in Tachyon: " + tfsRootPath);
      // }
    }

    // create the under FS handler (e.g. hdfs, local FS, s3 etc.)
    UnderFileSystem ufs = UnderFileSystem.get(ufsAddress);

    Queue<TachyonURI> ufsPathQueue = new LinkedList<TachyonURI>();
    if (excludePathPrefix.outList(ufsRootPath)) {
      ufsPathQueue.add(ufsAddrRootPath);
    }

    while (!ufsPathQueue.isEmpty()) {
      TachyonURI ufsPath = ufsPathQueue.poll(); // this is the absolute path
      LOG.info("Loading: " + ufsPath);
      if (ufs.isFile(ufsPath.toString())) {
        TachyonURI tfsPath = buildTFSPath(tachyonPath, ufsAddrRootPath, ufsPath);
        if (tfs.exist(tfsPath)) {
          LOG.info("File " + tfsPath + " already exists in Tachyon.");
          continue;
        }
        int fileId = tfs.createFile(tfsPath, ufsPath);
        if (fileId == -1) {
          LOG.info("Failed to create tachyon file: " + tfsPath);
        } else {
          LOG.info("Create tachyon file " + tfsPath + " with file id " + fileId + " and "
              + "checkpoint location " + ufsPath);
        }
      } else { // ufsPath is a directory
        String[] files = ufs.list(ufsPath.toString()); // ufs.list() returns relative path
        if (files != null) {
          for (String filePath : files) {
            LOG.info("Get: " + filePath);
            String aPath = CommonUtils.concat(ufsPath, filePath);
            String checkPath = aPath.substring(ufsAddrRootPath.toString().length());
View Full Code Here

  }

  @Test
  public void createFileWithUfsFileTest() throws IOException {
    String tempFolder = mLocalTachyonCluster.getTempFolderInUnderFs();
    UnderFileSystem underFs = UnderFileSystem.get(tempFolder);
    OutputStream os = underFs.create(tempFolder + "/temp", 100);
    os.close();
    TachyonURI uri = new TachyonURI("/abc");
    mTfs.createFile(uri, new TachyonURI(tempFolder + "/temp"));
    Assert.assertTrue(mTfs.exist(uri));
    Assert.assertEquals(tempFolder + "/temp", mTfs.getFile(uri).getUfsPath());
View Full Code Here

    mPreviousBlockOutStreams = new ArrayList<BlockOutStream>();
    mCachedBytes = 0;

    if (mWriteType.isThrough()) {
      mUnderFsFile = CommonUtils.concat(mTachyonFS.createAndGetUserUfsTempFolder(), mFile.mFileId);
      UnderFileSystem underfsClient = UnderFileSystem.get(mUnderFsFile, ufsConf);
      if (mBlockCapacityByte > Integer.MAX_VALUE) {
        throw new IOException("BLOCK_CAPCAITY (" + mBlockCapacityByte + ") can not bigger than "
            + Integer.MAX_VALUE);
      }
      mCheckpointOutputStream = underfsClient.create(mUnderFsFile, (int) mBlockCapacityByte);
    }
  }
View Full Code Here

      Boolean canComplete = false;
      if (mWriteType.isThrough()) {
        if (mCancel) {
          mCheckpointOutputStream.close();
          UnderFileSystem underFsClient = UnderFileSystem.get(mUnderFsFile);
          underFsClient.delete(mUnderFsFile, false);
        } else {
          mCheckpointOutputStream.flush();
          mCheckpointOutputStream.close();
          mTachyonFS.addCheckpoint(mFile.mFileId);
          canComplete = true;
View Full Code Here

  @Test
  public void RenameEditLogTest() throws Exception {
    String journalPrefix = "/tmp/JournalDir" + String.valueOf(System.currentTimeMillis());
    String journalPath = journalPrefix + "/log.data";
    String completedStr = journalPrefix + "/completed/";
    UnderFileSystem ufs = UnderFileSystem.get(journalPath);
    ufs.delete(journalPrefix, true);
    ufs.mkdirs(journalPrefix, true);
    OutputStream ops = ufs.create(journalPath);
    if (ops != null) {
      ops.close();
    }
    if (ufs != null) {
      ufs.close();
    }

    // Write operation and flush them to completed directory.
    EditLog log = new EditLog(journalPath, false, 0);
    log.setMaxLogSize(100);
    for (int i = 0; i < 124; i ++) {
      log.createFile(false, new TachyonURI("/sth" + i), false, Constants.DEFAULT_BLOCK_SIZE_BYTE,
          System.currentTimeMillis());
      log.flush();
    }
    log.close();

    // Rename completed edit logs when loading them.
    ufs = UnderFileSystem.get(completedStr);
    int numOfCompleteFiles = ufs.list(completedStr).length;
    Assert.assertTrue(numOfCompleteFiles > 0);
    EditLog.setBackUpLogStartNum(numOfCompleteFiles / 2);
    log = new EditLog(journalPath, false, 0);
    int numOfCompleteFilesLeft = numOfCompleteFiles - numOfCompleteFiles / 2 + 1;
    Assert.assertEquals(numOfCompleteFilesLeft, ufs.list(completedStr).length);
    for (int i = 0; i < numOfCompleteFilesLeft; i ++) {
      Assert.assertTrue(ufs.exists(completedStr + i + ".editLog"));
    }
    EditLog.setBackUpLogStartNum(-1);
    log.close();
    ufs.delete(journalPrefix, true);
    if (ufs != null) {
      ufs.close();
    }
  }
View Full Code Here

   * Create an empty file
   *
   * @throws IOException
   */
  public static void touch(String path) throws IOException {
    UnderFileSystem ufs = UnderFileSystem.get(path);
    OutputStream os = ufs.create(path);
    os.close();
  }
View Full Code Here

   * @throws IOException
   */
  boolean recache(int blockIndex) throws IOException {
    boolean succeed = true;
    String path = getUfsPath();
    UnderFileSystem underFsClient = UnderFileSystem.get(path);

    try {
      InputStream inputStream = underFsClient.open(path);

      long length = getBlockSizeByte();
      long offset = blockIndex * length;
      inputStream.skip(offset);

View Full Code Here

   */
  public static void create(MasterInfo info, String path) throws IOException {
    String tPath = path + ".tmp";
    String parentFolder = path.substring(0, path.lastIndexOf(TachyonURI.SEPARATOR));
    LOG.info("Creating the image file: " + tPath);
    UnderFileSystem ufs = UnderFileSystem.get(path);
    if (!ufs.exists(parentFolder)) {
      LOG.info("Creating parent folder " + parentFolder);
      ufs.mkdirs(parentFolder, true);
    }
    OutputStream os = ufs.create(tPath);
    DataOutputStream imageOs = new DataOutputStream(os);
    ObjectWriter writer = JsonObject.createObjectMapper().writer();

    info.writeImage(writer, imageOs);
    imageOs.flush();
    imageOs.close();

    LOG.info("Succefully created the image file: " + tPath);
    ufs.delete(path, false);
    ufs.rename(tPath, path);
    ufs.delete(tPath, false);
    LOG.info("Renamed " + tPath + " to " + path);
    // safe to close, nothing created here with scope outside function
    ufs.close();
  }
View Full Code Here

TOP

Related Classes of tachyon.UnderFileSystem

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.