Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.WriteOptions


      }
    }
  }

  public void createFile(Path p, int pri) throws Exception {
    WriteOptions options = new WriteOptions();
    options.setIoprio(NativeIO.IOPRIO_CLASS_BE, pri);
    OutputStream out = fs.create(p, null, true, 4096, (short) 3, BLOCK_SIZE,
        512, null, null, options);
    byte[] buffer = new byte[BLOCK_SIZE];
    out.write(buffer);
    out.close();
View Full Code Here


   */
  @Test
  public void testWriteOptions() {
    boolean overwrite = false;
    boolean forceSync = true;
    WriteOptions opt = CreateOptions.writeOptions(overwrite, forceSync);
    assertEquals(overwrite, ((WriteOptions)opt.getValue()).getOverwrite());
    assertEquals(forceSync, ((WriteOptions)opt.getValue()).getForceSync());
  }
View Full Code Here

  }
 
 
  @Test
  public void testWriteOptionsDefaultValue() {
    WriteOptions opt = CreateOptions.writeOptions(null, null);
    assertEquals(true, ((WriteOptions)opt.getValue()).getOverwrite());
    assertEquals(false, ((WriteOptions)opt.getValue()).getForceSync());
  }
View Full Code Here

    assertEquals(flag, options.getSyncFileRange());
  }

  @Test
  public void testSyncFileRange() {
    WriteOptions options = new WriteOptions();
    int a1 = NativeIO.SYNC_FILE_RANGE_WAIT_AFTER;
    int a2 = NativeIO.SYNC_FILE_RANGE_WAIT_BEFORE;
    int a3 = NativeIO.SYNC_FILE_RANGE_WRITE;

    // Default should be SYNC_FILE_RANGE_WRITE.
    assertEquals(a3, options.getSyncFileRange());

    check(a1, options);
    check(a2, options);
    check(a3, options);
    check(a1 | a2, options);
View Full Code Here

      long blockSize, Progressable progress, int buffersize,
      int bytesPerChecksum, boolean forceSync, boolean doParallelWrites,
      InetSocketAddress[] favoredNodes) throws IOException {
    return create(src, permission, overwrite, createParent, replication,
        blockSize, progress, buffersize, bytesPerChecksum, forceSync,
        doParallelWrites, favoredNodes, new WriteOptions());
  }
View Full Code Here

  /**
   * Creates a WriteOptions from given overwrite and forceSync values. If null passed,
   * it will use their default value.
   */
  public static WriteOptions writeOptions(Boolean overwrite, Boolean forceSync){
    WriteOptions wo = new WriteOptions();
    if (overwrite != null)
      wo.setOverwrite(overwrite);
    if (forceSync != null)
      wo.setForcesync(forceSync);
    return wo;
  }
View Full Code Here

 
  public FSDataOutputStream create(Path f, CreateOptions... opts) throws IOException {
    // Arguments Default values:
    FsPermission perms = (FsPermission) getOption(CreateOptions.Perms.class,
        FsPermission.getDefault(), opts);
    boolean overwrite = ((WriteOptions) getOption(WriteOptions.class, new WriteOptions(), opts))
        .getOverwrite();
    int bufferSize = (Integer) getOption(CreateOptions.BufferSize.class, getDefaultBufferSize(),
        opts);
    short replication = (Short) getOption(CreateOptions.ReplicationFactor.class,
        getDefaultReplication(), opts);
    long blockSize = (Long) getOption(CreateOptions.BlockSize.class, getDefaultBlockSize(), opts);
    boolean forceSync = ((WriteOptions) getOption(WriteOptions.class, new WriteOptions(), opts))
        .getForceSync();
    Progressable progress = (Progressable) getOption(CreateOptions.Progress.class, null, opts);

    if (forceSync)
      throw new IOException("Create force sync file is unsupported " + "for this filesystem"
View Full Code Here

      int bufferSize, short replication, long blockSize,
      Progressable progress, boolean forceSync, boolean doParallelWrites)
      throws IOException {
    return createNonRecursive(f, permission, overwrite, bufferSize,
        replication, blockSize, progress, forceSync, doParallelWrites,
        new WriteOptions());
  }
View Full Code Here

   
    WriteBlockHeader headerToReceive = new WriteBlockHeader(
        versionAndOpcode);
    headerToReceive.readFields(in);

    WriteOptions options = headerToReceive.getWritePipelineInfo()
        .getWriteOptions();
    boolean ioprioEnabled = !options.isIoprioDisabled();
    if (ioprioEnabled) {
      NativeIO.ioprioSetIfPossible(options.getIoprioClass(),
          options.getIoprioData());
    }

    int namespaceid = headerToReceive.getNamespaceId();
    Block block = new Block(headerToReceive.getBlockId(),
        dataXceiverServer.estimateBlockSize, headerToReceive.getGenStamp());
    if (LOG.isInfoEnabled()) {
      if (remoteAddress == null) {
        getAddresses();
      }
      LOG.info("Receiving block " + block +
               " src: " + remoteAddress +
               " dest: " + localAddress);
    }
    int pipelineSize = headerToReceive.getPipelineDepth(); // num of datanodes in entire pipeline
    boolean isRecovery = headerToReceive.isRecoveryFlag(); // is this part of recovery?
    String client = headerToReceive.getWritePipelineInfo().getClientName(); // working on behalf of this client
    boolean hasSrcDataNode = headerToReceive.getWritePipelineInfo()
        .hasSrcDataNode(); // is src node info present
    if (hasSrcDataNode) {
      srcDataNode = headerToReceive.getWritePipelineInfo().getSrcDataNode();
    }
    int numTargets = headerToReceive.getWritePipelineInfo().getNumTargets();
    DatanodeInfo targets[] = headerToReceive.getWritePipelineInfo().getNodes();
    int fadvise = headerToReceive.getWritePipelineInfo().getWriteOptions()
        .getFadvise();

    DataOutputStream mirrorOut = null// stream to next target
    DataInputStream mirrorIn = null;    // reply from next target
    DataOutputStream replyOut = null;   // stream to prev target
    Socket mirrorSock = null;           // socket to next target
    BlockReceiver blockReceiver = null; // responsible for data handling
    String mirrorNode = null;           // the name:port of next target
    String firstBadLink = "";           // first datanode that failed in connection setup

    updateCurrentThreadName("receiving block " + block + " client=" + client);
    InjectionHandler.processEvent(InjectionEvent.DATANODE_WRITE_BLOCK);
    try {
      boolean ifProfileEnabled = headerToReceive.getWritePipelineInfo()
          .getWriteOptions().ifProfileEnabled();
      boolean isSecondary = (targets.length + 1 != pipelineSize);
      // open a block receiver and check if the block does not exist
      blockReceiver = new BlockReceiver(namespaceid, block, block, in,
          s.getRemoteSocketAddress().toString(),
          s.getLocalSocketAddress().toString(),
          isRecovery, client, srcDataNode, datanode, isSecondary, fadvise,
          ifProfileEnabled, versionAndOpcode.getDataTransferVersion() >=
            DataTransferProtocol.PACKET_INCLUDE_VERSION_VERSION,
            options.getSyncFileRange());

      // get a connection back to the previous target
      replyOut = new DataOutputStream(new BufferedOutputStream(
                     NetUtils.getOutputStream(s, datanode.socketWriteTimeout),
                     SMALL_BUFFER_SIZE));
View Full Code Here

  /**
   * This test just triggers the codepath and doesn't do any other verification.
   */
  public void testSlowWritePacketLogging() throws Exception {
    DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
    WriteOptions options = new WriteOptions();
    options.setLogSlowWriteProfileDataThreshold(1);
    DFSWriteProfilingData data = new DFSWriteProfilingData();
    DFSClient.setProfileDataForNextOutputStream(data);
    OutputStream out = fs.create(new Path("/test"), null, true, 4096,
        (short) 3, 1024, 512, null, null, options);
    byte[] buffer = new byte[1024 * 1024];
 
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.WriteOptions

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.