Examples of org.apache.hadoop.fs.FSDataOutputStream$Buffer

org.apache.hadoop.fs.FSDataOutputStream
Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}, buffers output through a {@link BufferedOutputStream} and creates a checksumfile.

      System.out.println("data.length=" + data.length);
  
      //write data to a file
      final Path foo = new Path(dir, "foo" + n);
      {
        final FSDataOutputStream out = hdfs.create(foo, false, buffer_size,
            (short)2, block_size);
        out.write(data);
        out.close();
      }
      
      //compute checksum
      final FileChecksum hdfsfoocs = hdfs.getFileChecksum(foo);
      System.out.println("hdfsfoocs=" + hdfsfoocs);
      
      final FileChecksum hftpfoocs = hftp.getFileChecksum(foo);
      System.out.println("hftpfoocs=" + hftpfoocs);


      final Path qualified = new Path(hftpuri + dir, "foo" + n);
      final FileChecksum qfoocs = hftp.getFileChecksum(qualified);
      System.out.println("qfoocs=" + qfoocs);


      //write another file
      final Path bar = new Path(dir, "bar" + n);
      {
        final FSDataOutputStream out = hdfs.create(bar, false, buffer_size,
            (short)2, block_size);
        out.write(data);
        out.close();
      }
  
      { //verify checksum
        final FileChecksum barcs = hdfs.getFileChecksum(bar);
        final int barhashcode = barcs.hashCode();

View Full Code Here

      util.waitReplication(fs, topDir, (short)3);
      String outStr = runFsck(conf, 0, true, "/");
      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
      // Open a file for writing and do not close for now
      Path openFile = new Path(topDir + "/openFile");
      FSDataOutputStream out = fs.create(openFile);
      int writeCount = 0;
      while (writeCount != 100) {
        out.write(randomString.getBytes());
        writeCount++;                  
      }
      // We expect the filesystem to be HEALTHY and show one open file
      outStr = runFsck(conf, 0, true, topDir);
      System.out.println(outStr);
      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
      assertFalse(outStr.contains("OPENFORWRITE")); 
      // Use -openforwrite option to list open files
      outStr = runFsck(conf, 0, true, topDir, "-openforwrite");
      System.out.println(outStr);
      assertTrue(outStr.contains("OPENFORWRITE"));
      assertTrue(outStr.contains("openFile"));
      // Close the file
      out.close(); 
      // Now, fsck should show HEALTHY fs and should not show any open files
      outStr = runFsck(conf, 0, true, topDir);
      System.out.println(outStr);
      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
      assertFalse(outStr.contains("OPENFORWRITE"));

View Full Code Here

        // find user log directory
        Path userLogFile = 
          getJobHistoryLogLocationForUser(logFileName, jobConf);


        try{
          FSDataOutputStream out = null;
          PrintWriter writer = null;


          if (LOG_DIR != null) {
            // create output stream for logging in hadoop.job.history.location
            if (restarted) {
              logFile = recoverJobHistoryFile(jobConf, logFile);
              logFileName = logFile.getName();
            }
            
            int defaultBufferSize = 
              LOGDIR_FS.getConf().getInt("io.file.buffer.size", 4096);
            out = LOGDIR_FS.create(logFile, 
                            new FsPermission(HISTORY_FILE_PERMISSION),
                            true, 
                            defaultBufferSize, 
                            LOGDIR_FS.getDefaultReplication(), 
                            jobHistoryBlockSize, null);
            writer = new PrintWriter(out);
            fileManager.addWriter(jobId, writer);


            // cache it ...
            fileManager.setHistoryFile(jobId, logFile);
          }
          if (userLogFile != null) {
            // Get the actual filename as recoverJobHistoryFile() might return
            // a different filename
            userLogDir = userLogFile.getParent().toString();
            userLogFile = new Path(userLogDir, logFileName);
            
            // create output stream for logging 
            // in hadoop.job.history.user.location
            fs = userLogFile.getFileSystem(jobConf);
 
            out = fs.create(userLogFile, true, 4096);
            writer = new PrintWriter(out);
            fileManager.addWriter(jobId, writer);
          }
          
          ArrayList<PrintWriter> writers = fileManager.getWriters(jobId);
          // Log the history meta info
          JobHistory.MetaInfoManager.logMetaInfo(writers);


          String viewJobACL = "*";
          String modifyJobACL = "*";
          if (aclsEnabled) {
            viewJobACL = jobConf.get(JobACL.VIEW_JOB.getAclName(), " ");
            modifyJobACL = jobConf.get(JobACL.MODIFY_JOB.getAclName(), " ");
          }
          //add to writer as well 
          JobHistory.log(writers, RecordTypes.Job, 
                         new Keys[]{Keys.JOBID, Keys.JOBNAME, Keys.USER,
                                    Keys.SUBMIT_TIME, Keys.JOBCONF,
                                    Keys.VIEW_JOB, Keys.MODIFY_JOB,
                                    Keys.JOB_QUEUE}, 
                         new String[]{jobId.toString(), jobName, user, 
                                      String.valueOf(submitTime) , jobConfPath,
                                      viewJobACL, modifyJobACL,
                                      jobConf.getQueueName()}
                        ); 
             
        }catch(IOException e){
          LOG.error("Failed creating job history log file, disabling history", e);
          disableHistory = true; 
        }
      }
      // Always store job conf on local file system 
      String localJobFilePath =  JobInfo.getLocalJobFilePath(jobId); 
      File localJobFile = new File(localJobFilePath);
      FileOutputStream jobOut = null;
      try {
        jobOut = new FileOutputStream(localJobFile);
        jobConf.writeXml(jobOut);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Job conf for " + jobId + " stored at " 
                    + localJobFile.getAbsolutePath());
        }
      } catch (IOException ioe) {
        LOG.error("Failed to store job conf on the local filesystem ", ioe);
      } finally {
        if (jobOut != null) {
          try {
            jobOut.close();
          } catch (IOException ie) {
            LOG.info("Failed to close the job configuration file " 
                       + StringUtils.stringifyException(ie));
          }
        }
      }


      /* Storing the job conf on the log dir */
      Path jobFilePath = null;
      if (LOG_DIR != null) {
        jobFilePath = new Path(LOG_DIR + File.separator + 
                               jobUniqueString + "_conf.xml");
        fileManager.setConfFile(jobId, jobFilePath);
      }
      Path userJobFilePath = null;
      if (userLogDir != null) {
        userJobFilePath = new Path(userLogDir + File.separator +
                                   jobUniqueString + "_conf.xml");
      }
      FSDataOutputStream jobFileOut = null;
      try {
        if (LOG_DIR != null) {
          int defaultBufferSize = 
              LOGDIR_FS.getConf().getInt("io.file.buffer.size", 4096);
          if (!LOGDIR_FS.exists(jobFilePath)) {
            jobFileOut = LOGDIR_FS.create(jobFilePath, 
                                   new FsPermission(HISTORY_FILE_PERMISSION),
                                   true, 
                                   defaultBufferSize, 
                                   LOGDIR_FS.getDefaultReplication(), 
                                   LOGDIR_FS.getDefaultBlockSize(), null);
            jobConf.writeXml(jobFileOut);
            jobFileOut.close();
          }
        } 
        if (userLogDir != null) {
          fs = new Path(userLogDir).getFileSystem(jobConf);
          jobFileOut = fs.create(userJobFilePath);
          jobConf.writeXml(jobFileOut);
        }
        if (LOG.isDebugEnabled()) {
          LOG.debug("Job conf for " + jobId + " stored at " 
                    + jobFilePath + "and" + userJobFilePath );
        }
      } catch (IOException ioe) {
        LOG.error("Failed to store job conf in the log dir", ioe);
      } finally {
        if (jobFileOut != null) {
          try {
            jobFileOut.close();
          } catch (IOException ie) {
            LOG.info("Failed to close the job configuration file " 
                     + StringUtils.stringifyException(ie));
          }
        }

View Full Code Here

   * @param dirPath Path to create the data in.
   * @throws IOException If an error occurs creating the data.
   */
  public static void generateData(FileSystem fs, Path dirPath) 
      throws IOException {
    FSDataOutputStream out = fs.create(new Path(dirPath, "data.txt"));
    for (int i = 0; i < 10000; i++) {
      String line = generateRandomLine();
      out.write(line.getBytes("UTF-8"));
    }
    out.close();
  }

View Full Code Here

    filea = new Path(inputPath,"a");
    fileb = new Path(inputPath,"b");
    filec = new Path(inputPath,"c");
    archivePath = new Path(fs.getHomeDirectory(), "tmp");
    fs.mkdirs(inputPath);
    FSDataOutputStream out = fs.create(filea); 
    out.write("a".getBytes());
    out.close();
    out = fs.create(fileb);
    out.write("b".getBytes());
    out.close();
    out = fs.create(filec);
    out.write("c".getBytes());
    out.close();
  }

View Full Code Here

    }
  }
  
  void createFile(FileSystem fs, Path path, int fileLen) throws IOException {
    byte [] arr = new byte[fileLen];
    FSDataOutputStream out = fs.create(path);
    out.write(arr);
    out.close();
  }

View Full Code Here

          }
        }
      } catch (FileNotFoundException fnf) {
        // Lock file does not exist; try creating it ourselves
        try {
          FSDataOutputStream out = fs.create(lockPath, false);
          out.close();
          fs.setTimes(lockPath, -1, System.currentTimeMillis());
          // If we got here, we created the lock file successfully; go ahead
          // and launch the JobTracker
          Random r = new Random();
          String host = jtHosts[r.nextInt(jtHosts.length)];

View Full Code Here

    assertTrue("/ should be a directory",
      fileSystem.getFileStatus(path).isDir());


    // create a new file in the root, write data, do no close
    Path file1 = new Path("/unfinished-block");
    FSDataOutputStream out = fileSystem.create(file1);


    int writeSize = blockSize / 2;
    out.write(new byte[writeSize]);
    out.sync();
    
    FSDataInputStream in = fileSystem.open(file1);
    
    byte[] buf = new byte[4096];
    in.readFully(0, buf);
    in.close();


    waitForBlocks(fileSystem, file1, 1, writeSize);
    
    int blockMapSize = cluster.getDataNodes().get(0).blockScanner.blockMap.size();
    assertTrue(
      String.format("%d entries in blockMap and it should be empty", blockMapSize),
      blockMapSize == 0
    );
    
    
    out.close();
  }

View Full Code Here

  static final int numDatanodes = 3;
  short replication = 3;


  private void writeFile(FileSystem fileSys, Path name, int repl)
    throws IOException {
    FSDataOutputStream stm = fileSys.create(name, true,
                                            fileSys.getConf().getInt("io.file.buffer.size", 4096),
                                            (short)repl, (long)blockSize);
    byte[] buffer = new byte[fileSize];
    Random rand = new Random(seed);
    rand.nextBytes(buffer);
    stm.write(buffer);
    stm.close();
  }

View Full Code Here

    Path localPath = new Path(localPasswordFile);
    if (localFs.isFile(localPath)) {
      LOG.debug("Password file is already created by previous path");
      return;
    }
    FSDataOutputStream out = FileSystem.create(localFs, localPath,
        new FsPermission("400"));
    out.write(password);
    out.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.FSDataOutputStream$Buffer

com.alimama.mdrill.ui.service.utils.WebServiceParams

com.asakusafw.runtime.util.cache.HadoopFileCacheRepository

com.asakusafw.thundergate.runtime.cache.CacheStorageTest

com.hadoop.compression.lzo.LzoIndex

edu.umd.cloud9.io.FSProperty

ivory.core.util.CLIRUtils

org.apache.ambari.view.pig.resources.files.FileService

org.apache.ambari.view.pig.test.JobTest

org.apache.commons.collections15.buffer.TestBlockingBuffer

org.apache.commons.collections15.buffer.TestPredicatedBuffer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.