Examples of org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.fs.FSDataOutputStream
Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}, buffers output through a {@link BufferedOutputStream} and creates a checksumfile.

      // Insert them into a linked list.
      //
      for (int i = 0; i < numberOfFiles; i++) {
        short replication = (short)(AppendTestUtil.nextInt(numDatanodes) + 1);
        Path testFile = new Path("/" + i + ".dat");
        FSDataOutputStream stm = createFile(fs, testFile, replication);
        stm.close();
        testFiles.add(testFile);
      }


      // Create threads and make them run workload concurrently.
      workload = new Workload[numThreads];

View Full Code Here

    if (exists(f) && !overwrite) {
      throw new IOException("File already exists:"+f);
    }
    Path absolutePath = makeAbsolute(f);
    String key = pathToKey(absolutePath);
    return new FSDataOutputStream(new NativeS3FsOutputStream(getConf(), store,
        key, progress, bufferSize), statistics);
  }

View Full Code Here


          System.out.println("Workload thread " + id +
                             " appending " + sizeToAppend + " bytes " +
                             " to file " + testfile +
                             " of size " + len);
          FSDataOutputStream stm = fs.append(testfile);
          stm.write(fileContents, (int)len, sizeToAppend);
          stm.close();


          // wait for the file size to be reflected in the namenode metadata
          long startWaitTime = System.currentTimeMillis();
          while (fs.getFileStatus(testfile).getLen() != (len + sizeToAppend)) {
            try {

View Full Code Here

      tries = 40;
    }


    // Trying recovery
    boolean recovered = false;
    FSDataOutputStream out = null;
    while (!recovered && tries-- > 0) {
      try {
        out = fs.append(file1);
        LOG.info("Successfully opened for appends");
        recovered = true;
      } catch (IOException e) {
        if (!e.getMessage().contains("being recovered") &&
            !e.getMessage().contains("being created")) {
          throw e;
        }
        LOG.info("Failed open for append, waiting on lease recovery");
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ex) {
          // ignore it and try again
        }
      }
    }
    if (out != null) {
      out.close();
    }
    if (!recovered) {
      throw new RuntimeException("Recovery failed");
    }    
  }

View Full Code Here

    
    String src = "/file-1";
    String dst = "/file-2";
    Path srcPath = new Path(src);
    Path dstPath = new Path(dst);
    FSDataOutputStream fos = fs.create(srcPath);
   
    AppendTestUtil.write(fos, 0, writeSize);
    fos.sync();
    
    // renaming a file out from under a client will cause close to fail
    // and result in the lease remaining while the blocks are finalized on
    // the DNs
    fs.rename(srcPath, dstPath);


    try {
      fos.close();
      fail("expected IOException");
    } catch (IOException e) {
      //expected
    }

View Full Code Here

          AccessControlList acl = jobSubmitClient.getQueueAdmins(queue);
          jobCopy.set(QueueManager.toFullPropertyName(queue,
              QueueACL.ADMINISTER_JOBS.getAclName()), acl.getACLString());


          // Write job file to JobTracker's fs        
          FSDataOutputStream out = 
            FileSystem.create(fs, submitJobFile,
                new FsPermission(JobSubmissionFiles.JOB_FILE_PERMISSION));


          try {
            jobCopy.writeXml(out);
          } finally {
            out.close();
          }


          //
          // Now, actually submit the job (using the submit name)
          //

View Full Code Here


  private static MyFile createFile(Path root, FileSystem fs, int levels)
      throws IOException {
    MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
    Path p = new Path(root, f.getName());
    FSDataOutputStream out = fs.create(p);
    byte[] toWrite = new byte[f.getSize()];
    new Random(f.getSeed()).nextBytes(toWrite);
    out.write(toWrite);
    out.close();
    FileSystem.LOG.info("created: " + p + ", size=" + f.getSize());
    return f;
  }

View Full Code Here

                    getOutputCompressorClass(ctx, GzipCodec.class);


            CompressionCodec codec = ReflectionUtils.newInstance(codecClass, ctx.getConfiguration());
            Path file =  getDefaultWorkFile(ctx, ".nt"+codec.getDefaultExtension());
            FileSystem fs = file.getFileSystem(ctx.getConfiguration());
            FSDataOutputStream fileOut = fs.create(file, false);
            return codec.createOutputStream(fileOut);
        }
    }

View Full Code Here

        int result = new WindGateHadoopGet(conf).execute(buffer, testing.toString());
        assertThat(result, is(0));
    }


    private void put(Path path, String string) throws IOException {
        FSDataOutputStream out = fs.create(path, true);
        try {
            out.write(string.getBytes("UTF-8"));
        } finally {
            out.close();
        }
    }

View Full Code Here

                    getOutputCompressorClass(ctx, GzipCodec.class);


            CompressionCodec codec = ReflectionUtils.newInstance(codecClass, ctx.getConfiguration());
            Path file =  getDefaultWorkFile(ctx, ".nt"+codec.getDefaultExtension());
            FileSystem fs = file.getFileSystem(ctx.getConfiguration());
            FSDataOutputStream fileOut = fs.create(file, false);
            return new DataOutputStream(codec.createOutputStream(fileOut));
        }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.FSDataOutputStream

com.alimama.mdrill.ui.service.utils.WebServiceParams

com.asakusafw.runtime.util.cache.HadoopFileCacheRepository

com.asakusafw.thundergate.runtime.cache.CacheStorageTest

com.bah.geterdun.HeaderManagement

com.dianping.cat.hadoop.hdfs.UploaderAndCleaner

com.hadoop.compression.lzo.LzoIndex

edu.umd.cloud9.io.FSProperty

ivory.core.preprocess.BuildTermDocVectors$DocLengthDataWriterMapper

ivory.core.preprocess.BuildTermDocVectors$MyMapper

ivory.core.util.CLIRUtils

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.