Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FileStatus

3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
        throws IOException {
      // check if the tokenJob file is there..
      Path skPath = new Path(systemDirectory,
          jobId.toString()+"/"+TokenCache.JOB_TOKEN_HDFS_FILE);
     
      FileStatus status = null;
      long jobTokenSize = -1;
      status = systemFS.getFileStatus(skPath); //throws FileNotFoundException
      jobTokenSize = status.getLen();
     
      Path localJobTokenFile =
          lDirAlloc.getLocalPathForWrite(getLocalJobTokenFile(user,
              jobId.toString()), jobTokenSize, fConf);
   
View Full Code Here
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
                                   ") total_size(" + cbsize + ") listuri(" +
                                   srcfilelist + ")");
      }
      Path src = new Path(srcfilelist);
      FileSystem fs = src.getFileSystem(job);
      FileStatus srcst = fs.getFileStatus(src);

      ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
      LongWritable key = new LongWritable();
      FilePair value = new FilePair();
      final long targetsize = cbsize / numSplits;
      long pos = 0L;
      long last = 0L;
      long acc = 0L;
      long cbrem = srcst.getLen();
      SequenceFile.Reader sl = null;
      try {
        sl = new SequenceFile.Reader(fs, src, job);
        for (; sl.next(key, value); last = sl.getPosition()) {
          // if adding this split would put this split past the target size,
View Full Code Here
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
        if (!destFileSys.mkdirs(absdst.getParent())) {
          throw new IOException("Failed to create parent dir: " + absdst.getParent());
        }
        rename(tmpfile, absdst);

        FileStatus dststat = destFileSys.getFileStatus(absdst);
        if (dststat.getLen() != srcstat.getLen()) {
          destFileSys.delete(absdst, false);
          throw new IOException("File size not matched: copied "
              + bytesString(dststat.getLen()) + " to dst (=" + absdst
              + ") but expected " + bytesString(srcstat.getLen())
              + " from " + srcstat.getPath());       
        }
        updatePermissions(srcstat, dststat);
      }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
      JobID jobId, FileSystem fs, Configuration conf, Path jobSubmitDir)
  throws IOException {
    long maxMetaInfoSize = conf.getLong("mapreduce.jobtracker.split.metainfo.maxsize",
        10000000L);
    Path metaSplitFile = JobSubmissionFiles.getJobSplitMetaFile(jobSubmitDir);
    FileStatus fStatus = fs.getFileStatus(metaSplitFile);
    if (maxMetaInfoSize > 0 && fStatus.getLen() > maxMetaInfoSize) {
      throw new IOException("Split metadata size exceeded " +
          maxMetaInfoSize +". Aborting job " + jobId);
    }
    FSDataInputStream in = fs.open(metaSplitFile);
    byte[] header = new byte[JobSplit.META_SPLIT_FILE_HEADER.length];
View Full Code Here
539
540
541
542
543
544
545
546
547
548
549
     */
    public void map(LongWritable key,
                    FilePair value,
                    OutputCollector<WritableComparable<?>, Text> out,
                    Reporter reporter) throws IOException {
      final FileStatus srcstat = value.input;
      final Path relativedst = new Path(value.output);
      try {
        copy(srcstat, relativedst, out, reporter);
      } catch (IOException e) {
        ++failcount;
View Full Code Here
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
      // we do a recursive ls on these paths
      // and then write them to the input file
      // one at a time
      for (Path src: srcPaths) {
        ArrayList<FileStatusDir> allFiles = new ArrayList<FileStatusDir>();
        FileStatus fstatus = fs.getFileStatus(src);
        FileStatusDir fdir = new FileStatusDir(fstatus, null);
        recursivels(fs, fdir, allFiles);
        for (FileStatusDir statDir: allFiles) {
          FileStatus stat = statDir.getFileStatus();
          String toWrite = "";
          long len = stat.isDir()? 0:stat.getLen();
          if (stat.isDir()) {
            toWrite = "" + relPathToRoot(stat.getPath(), parentPath) + " dir ";
            //get the children
            FileStatus[] list = statDir.getChildren();
            StringBuffer sbuff = new StringBuffer();
            sbuff.append(toWrite);
            for (FileStatus stats: list) {
              sbuff.append(stats.getPath().getName() + " ");
            }
            toWrite = sbuff.toString();
          }
          else {
            toWrite +=  relPathToRoot(stat.getPath(), parentPath) + " file ";
          }
          srcWriter.append(new LongWritable(len), new
              Text(toWrite));
          srcWriter.sync();
          numFiles++;
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
      if (filelist != null) {
        buff.append(filelist.length).append("_");
        for (String s : filelist) {
          crc32.update(new String(s).getBytes());
          FileStatus fstatus=null;
          try {
            fstatus=this.fileStatus(s) ;
          } catch (Throwable e) {
            fstatus=null;
            logger.error("fileStatus", e);
          }
          if(fstatus!=null)
          {
            filesize += fstatus.getLen();
            filemodify = Math.max(filemodify, fstatus.getModificationTime());
          }

       
        }
      }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
      create();
  }

  boolean isDir = false;
  try {
      FileStatus status = fs.getFileStatus(directory);
      if (status != null) {
    isDir = status.isDir();
      }
  } catch (IOException e) {
      // file does not exist, isDir already set to false
  }
  if (!isDir) {
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
218
219
      fs.mkdirs(directory);
  }

  boolean isDir = false;
  try {
      FileStatus status = fs.getFileStatus(directory);
      if (status != null) {
    isDir = status.isDir();
      }
  } catch (IOException e) {
      // file does not exist, isDir already set to false
  }
  if (!isDir) {
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
269
    {
      boolean isnotrealtime=false;
      try{
      if(fs.exists(hdfsPartionShardPath))
      {
        FileStatus fstat=fs.getFileStatus(hdfsPartionShardPath);//针对云梯，要判断是文件还是目录
        if(!fstat.isDir())
        {
          isnotrealtime=true;
        }
      }else{
        isnotrealtime=true;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.FileStatus

  • com.alimama.mdrill.hdfsDirectory.FileSystemDirectory
  • com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil
  • com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest
  • com.asakusafw.windgate.hadoopfs.ssh.AbstractSshHadoopFsMirrorTest
  • com.cloudera.hoop.client.fs.TestHoopFileSystem
  • com.fb2pdf.hadoop.cluster.FB2StopwordsExcluder
  • com.hdfsTools.impl.checkImpl
  • com.yahoo.storm.yarn.Util
  • org.apache.accumulo.server.master.tableOps.BulkImport
  • org.apache.hadoop.filecache.TestTrackerDistributedCacheManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.