Examples of TachyonByteBuffer

tachyon.client.TachyonByteBuffer
TachyonByteBuffer is a wrapper on Java ByteBuffer plus some information needed by Tachyon.

Examples of tachyon.client.TachyonByteBuffer

    boolean pass = true;
    for (int i = 0; i < mNumFiles; i ++) {
      TachyonURI filePath = new TachyonURI(mFileFolder + "/part-" + i);
      LOG.debug("Reading data from {}", filePath);
      TachyonFile file = tachyonClient.getFile(filePath);
      TachyonByteBuffer buf = file.readByteBuffer(0);
      if (buf == null) {
        file.recache();
        buf = file.readByteBuffer(0);
      }
      buf.mData.order(ByteOrder.nativeOrder());
      for (int k = 0; k < mNumFiles; k ++) {
        pass = pass && (buf.mData.getInt() == k);
      }
      buf.close();
    }
    return pass;
  }

View Full Code Here

Examples of tachyon.client.TachyonByteBuffer

    boolean pass = true;
    LOG.debug("Reading data...");


    final long startTimeMs = CommonUtils.getCurrentMs();
    TachyonFile file = tachyonClient.getFile(mFilePath);
    TachyonByteBuffer buf = file.readByteBuffer(0);
    if (buf == null) {
      file.recache();
      buf = file.readByteBuffer(0);
    }
    buf.mData.order(ByteOrder.nativeOrder());
    for (int k = 0; k < mNumbers; k ++) {
      pass = pass && (buf.mData.getInt() == k);
    }
    buf.close();


    CommonUtils.printTimeTakenMs(startTimeMs, LOG, "readFile file " + mFilePath);
    return pass;
  }

View Full Code Here

Examples of tachyon.client.TachyonByteBuffer

      super(id, left, right, buf);
      mTC = TachyonFS.get(sMasterAddress);
    }


    public void readPartition() throws IOException {
      TachyonByteBuffer buf;
      if (sDebugMode) {
        LOG.info("Verifying the reading data...");


        for (int pId = mLeft; pId < mRight; pId ++) {
          TachyonFile file = mTC.getFile(new TachyonURI(sFileName + (pId + sBaseFileNumber)));
          buf = file.readByteBuffer(0);
          IntBuffer intBuf;
          intBuf = buf.mData.order(ByteOrder.nativeOrder()).asIntBuffer();
          for (int i = 0; i < sBlocskPerFile; i ++) {
            for (int k = 0; k < sBlockSizeBytes / 4; k ++) {
              int tmp = intBuf.get();
              if ((k == 0 && tmp == (i + mWorkerId)) || (k != 0 && tmp == k)) {
                LOG.debug("Partition at {} is {}", k, tmp);
              } else {
                throw new IllegalStateException("WHAT? " + tmp + " " + k);
              }
            }
          }
          buf.close();
        }
      }


      long sum = 0;
      if (sTachyonStreamingRead) {
        for (int pId = mLeft; pId < mRight; pId ++) {
          final long startTimeMs = System.currentTimeMillis();
          TachyonFile file = mTC.getFile(new TachyonURI(sFileName + (pId + sBaseFileNumber)));
          InputStream is = file.getInStream(ReadType.CACHE);
          long len = sBlocskPerFile * sBlockSizeBytes;


          while (len > 0) {
            int r = is.read(mBuf.array());
            len -= r;
            Preconditions.checkState(r != -1, "R == -1");
          }
          is.close();
          logPerIteration(startTimeMs, pId, "th ReadTachyonFile @ Worker ", pId);
        }
      } else {
        for (int pId = mLeft; pId < mRight; pId ++) {
          final long startTimeMs = System.currentTimeMillis();
          TachyonFile file = mTC.getFile(new TachyonURI(sFileName + (pId + sBaseFileNumber)));
          buf = file.readByteBuffer(0);
          for (int i = 0; i < sBlocskPerFile; i ++) {
            buf.mData.get(mBuf.array());
          }
          sum += mBuf.get(pId % 16);


          if (sDebugMode) {
            buf.mData.order(ByteOrder.nativeOrder()).flip();
            CommonUtils.printByteBuffer(LOG, buf.mData);
          }
          buf.mData.clear();
          logPerIteration(startTimeMs, pId, "th ReadTachyonFile @ Worker ", pId);
          buf.close();
        }
      }
      sResults[mWorkerId] = sum;
    }

View Full Code Here

Examples of tachyon.client.TachyonByteBuffer


    for (int column = 0; column < COLS; column ++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      TachyonFile tFile = rawColumn.getPartition(0);


      TachyonByteBuffer buf = tFile.readByteBuffer(0);
      if (buf == null) {
        tFile.recache();
        buf = tFile.readByteBuffer(0);
      }
      buf.mData.order(ByteOrder.nativeOrder());
      for (int k = 0; k < mDataLength; k ++) {
        pass = pass && (buf.mData.getInt() == k);
      }
      buf.close();
    }
    return pass;
  }

View Full Code Here

Examples of tachyon.client.TachyonByteBuffer

    }


    for (int k = 0; k < col; k ++) {
      RawColumn rawCol = table.getRawColumn(k);
      TachyonFile file = rawCol.getPartition(0, true);
      TachyonByteBuffer buf = file.readByteBuffer(0);
      Assert.assertEquals(TestUtils.getIncreasingByteBuffer(10), buf.mData);
      buf.close();
    }


    for (int k = 0; k < col; k ++) {
      RawColumn rawCol = table.getRawColumn(k);
      TachyonFile file = rawCol.getPartition(0, true);
      TachyonByteBuffer buf = file.readByteBuffer(0);
      Assert.assertEquals(TestUtils.getIncreasingByteBuffer(10), buf.mData);
      buf.close();
    }
  }

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.