Examples of org.apache.hadoop.fs.FileSystem.initialize()

org.apache.hadoop.fs.FileSystem.initialize()
Called after a new FileSystem instance is constructed. @param name a uri whose authority section names the host, port, etc.for this FileSystem @param conf the configuration

    FileSystem fs = null;
    Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
    if (clazz != null) {
      // This will be true for Hadoop 1.0, or 0.20.
      fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
      fs.initialize(uri, conf);
    } else {
      // For Hadoop 2.0, we have to go through FileSystem for the filesystem
      // implementation to be loaded by the service loader in case it has not
      // been loaded yet.
      Configuration clone = new Configuration(conf);

View Full Code Here

        if(isLocal) {
            fs = FileSystem.getLocal(new Configuration());
        } else {
            fs = new DistributedFileSystem();
            try {
                fs.initialize(new URI(hdfsUrl), new Configuration());
            } catch(URISyntaxException e) {
                throw new IllegalArgumentException(e);
            }
        }
        return fs;

View Full Code Here

        args.add(copySource);
        args.add(copyDest);


        console.printInfo("Copying " + copySource + " to " + copyDest);
        FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf);
        srcFs.initialize(sourceDir.toUri(), conf);


        FsShell fss = new FsShell(conf);
        int ret = 0;
        try {
          ret = ToolRunner.run(fss, args.toArray(new String[0]));

View Full Code Here

    Path HarPath = new Path(outDir,harDirName);
    if (fsDest.exists(HarPath)) {
      URI HarPathUri = HarPath.toUri();
      Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
      FileSystem fsHar = new HarFileSystem(fsDest);
      fsHar.initialize(inHarPath.toUri(), conf);
      FileStatus inHar = FileStatusCache.get(fsHar, inHarPath);
      if (inHar != null) {
        if (verifyParity(srcStatus, inHar, codec, conf)) {
          return new ParityFilePair(inHarPath, inHar, fsHar);
        }

View Full Code Here

    Path HarPath = new Path(outDir,harDirName);
    if (fsDest.exists(HarPath)) {  
      URI HarPathUri = HarPath.toUri();
      Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
      FileSystem fsHar = new HarFileSystem(fsDest);
      fsHar.initialize(inHarPath.toUri(), conf);
      if (fsHar.exists(inHarPath)) {
        FileStatus inHar = fsHar.getFileStatus(inHarPath);
        if (inHar.getModificationTime() == srcStatus.getModificationTime()) {
          return new ParityFilePair(inHarPath,fsHar);
        }

View Full Code Here


    FileSystem fsHar = new HarFileSystem(destFs);
    String harURIPath = harPath.toUri().getPath();
    Path qualifiedPath = new Path("har://", harURIPath +
      Path.SEPARATOR + harPath.getParent().toUri().getPath());
    fsHar.initialize(qualifiedPath.toUri(), conf);
    FileStatus[] filesInHar = fsHar.listStatus(qualifiedPath);
    if (filesInHar.length == 0) {
      return 0;
    }
    int numUseless = 0;

View Full Code Here

                         stripeLength, corruptOffset);
            if (npath == null)
              continue;


            FileSystem fs1 = getUnderlyingFileSystem(conf);
            fs1.initialize(npath.toUri(), conf);
            LOG.info("Opening alternate path " + npath + " at offset " + curpos);
            FSDataInputStream fd = fs1.open(npath, buffersize);
            fd.seek(curpos);
            underLyingStream.close();
            underLyingStream = fd;

View Full Code Here

        args.add(copySource);
        args.add(copyDest);


        console.printInfo("Copying " + copySource + " to " + copyDest);
        FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf);
        srcFs.initialize(sourceDir.toUri(), conf);


        FsShell fss = new FsShell(conf);
        int ret = 0;
        try {
          ret = ToolRunner.run(fss, args.toArray(new String[0]));

View Full Code Here

        args.add(copySource);
        args.add(copyDest);


        console.printInfo("Copying " + copySource + " to " + copyDest);
        FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf);
        srcFs.initialize(sourceDir.toUri(), conf);


        FsShell fss = new FsShell(conf);
        int ret = 0;
        try {
          ret = ToolRunner.run(fss, args.toArray(new String[0]));

View Full Code Here

    FileSystem fs = null;
    Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
    if (clazz != null) {
      // This will be true for Hadoop 1.0, or 0.20.
      fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
      fs.initialize(uri, conf);
    } else {
      // For Hadoop 2.0, we have to go through FileSystem for the filesystem
      // implementation to be loaded by the service loader in case it has not
      // been loaded yet.
      Configuration clone = new Configuration(conf);

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.