Examples of org.apache.hadoop.fs.FileSystem.open()

org.apache.hadoop.fs.FileSystem.open()
Opens an FSDataInputStream at the indicated Path. @param f the file name to open @param bufferSize the size of the buffer to be used.

    final Collection<Path> results = new PrettyPrintList<Path>(new ListPrinter<Path>() {
      @Override
      public String toString(Path e) throws IOException {
        try {
          final FileSystem srcFS = getFS(e);
          return getContent(srcFS.open(e));
        } catch (IOException ex) {
          return "No such file or directory " + e.toUri();
        }
      }
    });

View Full Code Here

        Path srcPat = new Path(uri);
        FileSystem srcFs = getFS(srcPat);


        for (Path src : FileUtil.stat2Paths(srcFs.globStatus(srcPat), srcPat)) {
          Assert.isTrue(srcFs.isFile(src), "Source must be a file");
          i = srcFs.open(src);
          switch (i.readShort()) {
          case 0x1f8b: // RFC 1952
            i.seek(0);
            in = new GZIPInputStream(i);
            break;

View Full Code Here

            end = start + split.getLength();
            final Path file = split.getPath();
 
            // open the file and seek to the start of the split
            FileSystem fs = file.getFileSystem(job);
            FSDataInputStream fileIn = fs.open(split.getPath());
            
            // Seek to the start of the file
            fileIn.seek(start);
        
            if(file.toString().endsWith(".bz2") || file.toString().endsWith(".bz"))

View Full Code Here

        FSDataInputStream input = null;


        Configuration config = new Configuration();


        FileSystem fs = source.getFileSystem(config);
        input = fs.open(source);


        while(true) {
            int read = input.read(buffer);
            if(read < 0) {
                break;

View Full Code Here


    public static Props readProps(String file) throws IOException {
        Path path = new Path(file);
        FileSystem fs = path.getFileSystem(new Configuration());
        if(fs.exists(path)) {
            InputStream input = fs.open(path);
            try {
                // wrap it up in another layer so that the user can override
                // properties
                Props p = new Props(null, input);
                return new Props(p);

View Full Code Here

    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);


    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {

View Full Code Here

    if (!isRead) {
      Path file = split.getPath();
      logger.debug("Path is " + file);
      fileName = file.getName();
      FileSystem fs = file.getFileSystem(context.getConfiguration());
      stream = fs.open(file);
      logger.debug("Opened stream");
      isRead = true;
      return true;
    }
    return false;

View Full Code Here

    expectedOutput.add("/iDELIM9000");
    expectedOutput.add("/jDELIM10000");
    int count = 0;
    for (FileStatus fileStat: status) {
      logger.debug("File status is " + fileStat.getPath());
      FSDataInputStream in = outputFS.open(fileStat.getPath());
      String line = null;      
      while ((line = in.readLine()) != null) {
        logger.debug("Output is " + line);
        assertTrue("Matched output " + line , expectedOutput.contains(line));
        expectedOutput.remove(line);

View Full Code Here

    expectedOutput.add("/iDELIM9000DELIM10");
    expectedOutput.add("/jDELIM10000DELIM10");
    int count = 0;
    for (FileStatus fileStat: status) {
      logger.debug("File status is " + fileStat.getPath());
      FSDataInputStream in = outputFS.open(fileStat.getPath());
      String line = null;      
      while ((line = in.readLine()) != null) {
        logger.debug("Output is " + line);
        assertTrue("Matched output " + line , expectedOutput.contains(line));
        expectedOutput.remove(line);

View Full Code Here

            LOGGER.info("Importing dictionary file: " + fileStatus.getPath().toString());
            String conceptName = FilenameUtils.getBaseName(fileStatus.getPath().toString());
            conceptName = URLDecoder.decode(conceptName, "UTF-8");
            Concept concept = ontologyRepository.getConceptByIRI(conceptName);
            checkNotNull(concept, "Could not find concept with name " + conceptName);
            writeFile(fs.open(fileStatus.getPath()), conceptName, user);
        }


        modelSession.close();
        return 0;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.