Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataInputStream$Buffer


    long maxsize = 1024l * 1024 * 1024 * 100;
   
 
    for (FileStatus f : filelist) {
      if (!f.isDir() && !f.getPath().getName().startsWith("_")) {
        FSDataInputStream in = fs.open(f.getPath());
        BufferedReader bf=new BufferedReader(new InputStreamReader(in));
        String line;
        while ((line = bf.readLine()) != null) {
          bytesRead += line.getBytes().length;
          String towrite=line;
          towrite=line.replaceAll(",", "_").replaceAll("\001", ",").replaceAll("\t", ",").replaceAll("\"", "");
          if(!towrite.isEmpty()&&towrite.indexOf("rep0")>=0)
          {

            String[] cols=towrite.split(",");
            for(String s:cols)
            {
              if(s.indexOf("@abtest@")>=0)
              {
                String[] sss=s.split("@abtest@");
                if(sss.length>1)
                {
                 
                  outputA.writeUTF(sss[1]+"\r\n");
                break;
                }

              }
            }
         
          }
         
          if(!towrite.isEmpty()&&towrite.indexOf("rep1")>=0)
          {

            String[] cols=towrite.split(",");
            for(String s:cols)
            {
              if(s.indexOf("@abtest@")>=0)
              {
                String[] sss=s.split("@abtest@");
                if(sss.length>1)
                {
                  outputB.writeUTF(sss[1]+"\r\n");
                break;
                }

              }
            }
         
          }
          if (bytesRead >= maxsize) {
            bf.close();
            in.close();
            return;
          }
        }
        bf.close();
        in.close();
      }
    }
   
     outputA.close();
     outputB.close();
View Full Code Here


    Long bytesRead = 0l;
    long maxsize = 1024l * 1024 * 1024 * 10;
 
    for (FileStatus f : filelist) {
      if (!f.isDir() && !f.getPath().getName().startsWith("_")) {
        FSDataInputStream in = fs.open(f.getPath());
        BufferedReader bf=new BufferedReader(new InputStreamReader(in));
        String line;
        while ((line = bf.readLine()) != null) {
          bytesRead += line.getBytes().length;
          String towrite=line;
          towrite=line.replaceAll(",", "_").replaceAll("\001", ",").replaceAll("\t", ",").replaceAll("\"", "");
          if(!towrite.isEmpty()&&towrite.indexOf(match)>=0)
          {
           
            String label="A";
            if(towrite.indexOf("rep0")>=0)
            {
              label="A";
            }
            if(towrite.indexOf("rep1")>=0)
            {
              label="B";
            }
           
            if(onlycustid)
            {
              String[] cols=towrite.split(",");
              for(String s:cols)
              {
                if(s.indexOf("@abtest@")>=0)
                {
                  String[] sss=s.split("@abtest@");
                  if(sss.length>1)
                  {
                  outStream.write(sss[1]+","+label);
                  outStream.write("\r\n");
                  break;
                  }

                }
              }
            }else{
              outStream.write(towrite);
              outStream.write("\r\n")
            }
          }
          if (bytesRead >= maxsize) {
            bf.close();
            in.close();
            return;
          }
        }
        bf.close();
        in.close();
      }
    }
    return;
  }
View Full Code Here

    Long bytesRead = 0l;
    long maxsize = 1024l * 1024 * 1024 * 10;
 
    for (FileStatus f : filelist) {
      if (!f.isDir() && !f.getPath().getName().startsWith("_")) {
        FSDataInputStream in = fs.open(f.getPath());
        BufferedReader bf=new BufferedReader(new InputStreamReader(in));
        String line;
        while ((line = bf.readLine()) != null) {
          bytesRead += line.getBytes().length;
          String towrite=line;
          if(line.indexOf("\001")>=0)
          {
            towrite=line.replaceAll(",", "_").replaceAll("\001", ",").replaceAll("\t", "_").replaceAll("\"", "");
          }else if(line.indexOf("\t")>=0)
          {
            towrite=line.replaceAll(",", "_").replaceAll("\001", "_").replaceAll("\t", ",").replaceAll("\"", "");
          }
          if(!towrite.isEmpty())
          {
          outStream.write(towrite);
          outStream.write("\r\n");
          }
          if (bytesRead >= maxsize) {
            bf.close();
            in.close();
            return;
          }
        }
        bf.close();
        in.close();
      }
    }
    return;
  }
View Full Code Here

        Configuration conf = new Configuration();
        Path p = new Path(file);
        FileSystem fs = p.getFileSystem(conf);
        if(fs.exists(p))
        {
          FSDataInputStream in = fs.open(p);
          BufferedReader br = new BufferedReader(new InputStreamReader(in));
          String s1 = null;
          while ((s1 = br.readLine()) != null) {
            String line = s1.trim();
            if (!line.isEmpty()) {
              set.add(line);
            }
          }
          br.close();
          in.close();
        }
        match.put(file, set);
      } catch (IOException e) {
      }
     
View Full Code Here

    connection = openConnection("/data" + f.toUri().getPath(),
        "ugi=" + getUgiParameter());
    connection.setRequestMethod("GET");
    connection.connect();
    final InputStream in = connection.getInputStream();
    return new FSDataInputStream(new FSInputStream() {
        public int read() throws IOException {
          return in.read();
        }
        public int read(byte[] b, int off, int len) throws IOException {
          return in.read(b, off, len);
View Full Code Here

   * @param fs FileSystem where history file is present
   * @throws IOException
   */
  public static void parseHistoryFromFS(String path, Listener l, FileSystem fs)
  throws IOException{
    FSDataInputStream in = fs.open(new Path(path));
    BufferedReader reader = new BufferedReader(new InputStreamReader (in));
    try {
      String line = null;
      StringBuffer buf = new StringBuffer();
     
View Full Code Here

      return null;
    }
   
    if (active) {
      try {
        FSDataInputStream dataIn = getJobInfoFile(jobId);
        if (dataIn != null) {
          jobStatus = readJobStatus(dataIn);
          dataIn.close();
        }
      } catch (IOException ex) {
        LOG.warn("Could not read [" + jobId + "] job status : " + ex, ex);
      }
    }
View Full Code Here

   */
  public JobProfile readJobProfile(JobID jobId) {
    JobProfile jobProfile = null;
    if (active) {
      try {
        FSDataInputStream dataIn = getJobInfoFile(jobId);
        if (dataIn != null) {
          readJobStatus(dataIn);
          jobProfile = readJobProfile(dataIn);
          dataIn.close();
        }
      } catch (IOException ex) {
        LOG.warn("Could not read [" + jobId + "] job profile : " + ex, ex);
      }
    }
View Full Code Here

   */
  public Counters readCounters(JobID jobId) throws AccessControlException {
    Counters counters = null;
    if (active) {
      try {
        FSDataInputStream dataIn = getJobInfoFile(jobId);
        if (dataIn != null) {
          JobStatus jobStatus = readJobStatus(dataIn);
          JobProfile profile = readJobProfile(dataIn);
          String queue = profile.getQueueName();
          // authorize the user for job view access
          aclsManager.checkAccess(jobStatus,
              UserGroupInformation.getCurrentUser(), queue,
              Operation.VIEW_JOB_COUNTERS);

          counters = readCounters(dataIn);
          dataIn.close();
        }
      } catch (AccessControlException ace) {
        throw ace;
      } catch (IOException ex) {
        LOG.warn("Could not read [" + jobId + "] job counters : " + ex, ex);
View Full Code Here

                                                               int fromEventId,
                                                               int maxEvents) {
    TaskCompletionEvent[] events = TaskCompletionEvent.EMPTY_ARRAY;
    if (active) {
      try {
        FSDataInputStream dataIn = getJobInfoFile(jobId);
        if (dataIn != null) {
          readJobStatus(dataIn);
          readJobProfile(dataIn);
          readCounters(dataIn);
          events = readEvents(dataIn, fromEventId, maxEvents);
          dataIn.close();
        }
      } catch (IOException ex) {
        LOG.warn("Could not read [" + jobId + "] job events : " + ex, ex);
      }
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.FSDataInputStream$Buffer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.