Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataOutputStream$Buffer


        localFs.rename(indexFileName[0], finalIndexFile);
        return;
      }
     
      //The output stream for the final single output file
      FSDataOutputStream finalOut = localFs.create(finalOutputFile, true,
                                                   4096);
      //The final index file output stream
      FSDataOutputStream finalIndexOut = localFs.create(finalIndexFile, true,
                                                        4096);
      long segmentStart;
     
      if (numSpills == 0) {
        //create dummy files
        for (int i = 0; i < partitions; i++) {
          segmentStart = finalOut.getPos();
          Writer writer = SequenceFile.createWriter(job, finalOut,
                                                    job.getMapOutputKeyClass(),
                                                    job.getMapOutputValueClass(),
                                                    compressionType, codec);
          finalIndexOut.writeLong(segmentStart);
          finalIndexOut.writeLong(finalOut.getPos() - segmentStart);
          writer.close();
        }
        finalOut.close();
        finalIndexOut.close();
        return;
      }
      {
        //create a sorter object as we need access to the SegmentDescriptor
        //class and merge methods
        Sorter sorter = new Sorter(localFs, job.getOutputKeyComparator(), valClass, job);
        sorter.setProgressable(reporter);
       
        for (int parts = 0; parts < partitions; parts++){
          List<SegmentDescriptor> segmentList =
            new ArrayList<SegmentDescriptor>(numSpills);
          for(int i = 0; i < numSpills; i++) {
            FSDataInputStream indexIn = localFs.open(indexFileName[i]);
            indexIn.seek(parts * 16);
            long segmentOffset = indexIn.readLong();
            long segmentLength = indexIn.readLong();
            indexIn.close();
            SegmentDescriptor s = sorter.new SegmentDescriptor(segmentOffset,
                                                               segmentLength, filename[i]);
            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
          //partition data don't span partition boundaries
          writer.close();
          //when we write the offset/length to the final index file, we write
          //longs for both. This helps us to reliably seek directly to the
          //offset/length for a partition when we start serving the byte-ranges
          //to the reduces. We probably waste some space in the file by doing
          //this as opposed to writing VLong but it helps us later on.
          finalIndexOut.writeLong(segmentStart);
          finalIndexOut.writeLong(finalOut.getPos()-segmentStart);
        }
        finalOut.close();
        finalIndexOut.close();
        //cleanup
        for(int i = 0; i < numSpills; i++) {
          localFs.delete(filename[i]);
          localFs.delete(indexFileName[i]);
        }
View Full Code Here


      }

      Path tmpfile = new Path(job.get(TMP_DIR_LABEL), relativedst);
      long cbcopied = 0L;
      FSDataInputStream in = null;
      FSDataOutputStream out = null;
      try {
        // open src file
        in = srcstat.getPath().getFileSystem(job).open(srcstat.getPath());
        reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());
        // open tmp file
        out = preserve_status
          ? destFileSys.create(tmpfile, true, sizeBuf, srcstat.getReplication(),
             srcstat.getBlockSize(), reporter)
          : destFileSys.create(tmpfile, reporter);
        // copy file
        int cbread;
        while ((cbread = in.read(buffer)) >= 0) {
          out.write(buffer, 0, cbread);
          cbcopied += cbread;
          reporter.setStatus(pcntfmt.format(100.0 * cbcopied / srcstat.getLen())
              + " " + absdst + " [ " +
              StringUtils.humanReadableInt(cbcopied) + " / " +
              StringUtils.humanReadableInt(srcstat.getLen()) + " ]");
View Full Code Here

    Path regioninfo = new Path(this.regiondir, REGIONINFO_FILE);
    if (this.fs.exists(regioninfo) &&
        this.fs.getFileStatus(regioninfo).getLen() > 0) {
      return;
    }
    FSDataOutputStream out = this.fs.create(regioninfo, true);
    try {
      this.regionInfo.write(out);
      out.write('\n');
      out.write('\n');
      out.write(Bytes.toBytes(this.regionInfo.toString()));
    } finally {
      out.close();
    }
  }
View Full Code Here

                                     ie);
        }
      }
    });
    // write the splits to a file for the job tracker
    FSDataOutputStream out = FileSystem.create(fs,
        submitSplitFile, new FsPermission(JOB_FILE_PERMISSION));
    try {
      writeSplitsFile(splits, out);
    } finally {
      out.close();
    }
    job.set("mapred.job.split.file", submitSplitFile.toString());
    job.setNumMapTasks(splits.length);
       
    // Write job file to JobTracker's fs       
    out = FileSystem.create(fs, submitJobFile,
        new FsPermission(JOB_FILE_PERMISSION));

    try {
      job.write(out);
    } finally {
      out.close();
    }

    //
    // Now, actually submit the job (using the submit name)
    //
View Full Code Here

    Configuration conf = new Configuration(util.getConfiguration());

    // populate input file
    FileSystem fs = FileSystem.get(conf);
    Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat"));
    FSDataOutputStream op = fs.create(inputPath, true);
    if (data == null) {
      data = "KEY\u001bVALUE1\u001bVALUE2\n";
    }
    op.write(Bytes.toBytes(data));
    op.close();
    LOG.debug(String.format("Wrote test data to file: %s", inputPath));

    if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
      LOG.debug("Forcing combiner.");
      conf.setInt("min.num.spills.for.combine", 1);
View Full Code Here

      FileSystem fs = region.getRegionFileSystem().getFileSystem();

      Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
      for (int i = 1000; i < 1050; i += 10) {
        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
        FSDataOutputStream dos = fs.create(recoveredEdits);
        dos.writeInt(i);
        dos.close();
      }
      long minSeqId = 2000;
      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
      FSDataOutputStream dos = fs.create(recoveredEdits);
      dos.close();

      Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
      for (Store store : region.getStores().values()) {
        maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
      }
View Full Code Here

    public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job,
        String name, Progressable progress) throws IOException {

      Path dir = getWorkOutputPath(job);
      FileSystem fs = dir.getFileSystem(job);
      FSDataOutputStream fileOut = fs.create(new Path(dir, name), progress);
      return new LineRecordWriter<K, V>(fileOut);
    }
View Full Code Here

    }

    public void createFile(String file, String content) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
        byte[] buff = content.getBytes();
        FSDataOutputStream os = null;
        try {
            os = fs.create(new Path(file));
            os.write(buff, 0, buff.length);
            System.out.println("Create: " + file);
        } finally {
            if (os != null)
                os.close();
        }
        fs.close();
    }
View Full Code Here

    // is set to roughly same count as expected number of writers
    // and strategy init is accurate enough to find a good starting
    // position for naming, we should always get a next available
    // path and its stream.
    Path p = null;
    FSDataOutputStream wout = null;
    int openAttempt = 0;
    do {
      try {
        p = getResolvedPath();
        if (isAppendable() && p.getFileSystem(getConfiguration()).exists(p)) {
View Full Code Here

    assertThat(context.getLong(DataStoreItemReader.READ_POSITION), is(300l));
  }

  private void createTestData(Path path) throws IOException {
    FileSystem fs = FileSystem.get(getYarnCluster().getConfiguration());
    FSDataOutputStream out = fs.create(path);
    for (int i = 0; i < 300; i++) {
      out.writeBytes("line" + i + "\n");
    }
    out.close();
    assertTrue(fs.exists(path));
    assertThat(fs.getFileStatus(path).getLen(), greaterThan(0l));
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.FSDataOutputStream$Buffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.