List of copyFromLocalFile() Examples

Examples of copyFromLocalFile()

org.apache.hadoop.fs.FileSystem.copyFromLocalFile()
The src file is on the local disk. Add it to FS at the given dst name and the source is kept intact afterwards

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

        HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
                (HadoopApplicationDeploymentDescriptionType)appDepDesc;
        if(appDepDesc.isSetInputDataDirectory() && isInputDataDirectoryLocal(appDepDesc)){
            Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
            FileSystem hdfs = FileSystem.get(hadoopConf);
            hdfs.copyFromLocalFile(new Path(appDepDesc.getInputDataDirectory()),
                    new Path(hadoopAppDesc.getHadoopJobConfiguration().getHdfsInputDirectory()));
        }
    }


    private boolean isInputDataDirectoryLocal(ApplicationDeploymentDescriptionType appDepDesc){

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      // use jar name if job is not named. 
      if ("".equals(job.getJobName())){
        job.setJobName(new Path(originalJarPath).getName());
      }
      job.setJar(submitJarFile.toString());
      fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
      fs.setReplication(submitJarFile, replication);
      fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION));
    } else {
      LOG.warn("No job jar file set.  User classes may not be found. "+
               "See JobConf(Class) or JobConf#setJar(String).");

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

            JobConf hadoopConfig = HdpBootstrap.hadoopConfig();
            FileSystem fs = FileSystem.get(hadoopConfig);
            if (!(fs instanceof LocalFileSystem)) {
                Path src = new Path(localPath);
                Path dst = new Path(destination);
                fs.copyFromLocalFile(false, true, src, dst);
                System.out.println(String.format("Copying [%s] to [%s]", src, dst));
            }
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

        fs.close();
    }


    public void copyFile(String local, String remote) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
        fs.copyFromLocalFile(new Path(local), new Path(remote));
        System.out.println("copy from: " + local + " to " + remote);
        fs.close();
    }


    public void download(String remote, String local) throws IOException {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

    List<Path> srcs = (List<Path>) va[0];
    Path dstPath = (Path) va[1];


    try {
      FileSystem dstFs = getFS(dstPath);
      dstFs.copyFromLocalFile(false, false, srcs.toArray(new Path[srcs.size()]), dstPath);
    } catch (IOException ex) {
      throw new HadoopException("Cannot copy resources " + ex.getMessage(), ex);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      doclistcache=new DocumentList();
      shardWriter.optimize();
      shardWriter.close();
      Configuration conf = context.getConfiguration();
      FileSystem fs = FileSystem.get(conf);
      fs.copyFromLocalFile(new Path(localtmpath), new Path(tmpath));


      if (!fs.exists(new Path(indexHdfsPath))) {
        fs.rename(new Path(tmpath), new Path(indexHdfsPath));
      }
      if (shardWriter.getNumDocs() > 0 && lastkey != null) {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      e.getValue().synctxid();
      txid=Math.max(txid, e.getValue().readTxid());
      Path local=new Path(e.getKey());
      Path dst=new Path(tmpDir,local.getName());
      LOG.info("####copyFromLocalFile####"+local.toString()+","+dst.toString());
      fs.copyFromLocalFile(local,dst);
    }
    
    Path realtimefinal=new Path(params.hdfsPath,"realtime");
    if(fs.exists(realtimefinal))
    {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      e.getValue().synctxid();
      txid=Math.max(txid, e.getValue().readTxid());
      Path local=new Path(e.getKey());
      Path dst=new Path(tmpDir,local.getName());
      LOG.info("####copyFromLocalFile####"+local.toString()+","+dst.toString());
      fs.copyFromLocalFile(local,dst);
    }
    
    Path realtimefinal=new Path(params.hdfsPath,"realtime");
    if(fs.exists(realtimefinal))
    {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

      if ("".equals(job.getJobName())){
        job.setJobName(new Path(originalJarPath).getName());
      }
      Path submitJarFile = JobSubmissionFiles.getJobJar(submitJobDir);
      job.setJar(submitJarFile.toString());
      fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
      fs.setReplication(submitJarFile, replication);
      fs.setPermission(submitJarFile, 
          new FsPermission(JobSubmissionFiles.JOB_FILE_PERMISSION));
    } else {
      LOG.warn("No job jar file set.  User classes may not be found. "+

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

          + hdfs.getUri().getPort());
      proxyConf.set("hdfsproxy.https.address", "localhost:0");
      final String namenode = hdfs.getUri().toString();
      if (namenode.startsWith("hdfs://")) {
        MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR + "/srcdat");
        hdfs.copyFromLocalFile
      (new Path("file:///" + TEST_ROOT_DIR + "/srcdat"),
             new Path(namenode + "/destdat" ));
        assertTrue("Source and destination directories do not match.",
            checkFiles(hdfs, "/destdat", files));

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.