Examples of org.apache.hadoop.util.LineReader

org.apache.hadoop.util.LineReader
A class that provides a line reader from an input stream. Depending on the constructor used, lines will either be terminated by:
- one of the following: '\n' (LF) , '\r' (CR), or '\r\n' (CR+LF).
- or, a custom byte sequence delimiter
In both cases, EOF also terminates an otherwise unterminated line.

    final FileInputStream in2 = new FileInputStream(fnLocal2.toString());
    assertEquals("concat bytes available", 2734, in1.available());
    assertEquals("concat bytes available", 3413, in2.available()); // w/hdr CRC


    CompressionInputStream cin2 = gzip.createInputStream(in2);
    LineReader in = new LineReader(cin2);
    Text out = new Text();


    int numBytes, totalBytes=0, lineNum=0;
    while ((numBytes = in.readLine(out)) > 0) {
      ++lineNum;
      totalBytes += numBytes;
    }
    in.close();
    assertEquals("total uncompressed bytes in concatenated test file",
                 5346, totalBytes);
    assertEquals("total uncompressed lines in concatenated test file",
                 84, lineNum);

View Full Code Here

   * @throws Exception
   */
  public static void main(String[] args) throws Exception {
    for(String arg: args) {
      System.out.println("Working on " + arg);
      LineReader reader = makeStream(unquote(arg));
      Text line = new Text();
      int size = reader.readLine(line);
      while (size > 0) {
        System.out.println("Got: " + line.toString());
        size = reader.readLine(line);
      }
      reader.close();
    }
  }

View Full Code Here

    }


  }


  private static LineReader makeStream(String str) throws IOException {
    return new LineReader(new ByteArrayInputStream
                                             (str.getBytes("UTF-8")), 
                                           defaultConf);
  }

View Full Code Here

    return new LineReader(new ByteArrayInputStream
                                             (str.getBytes("UTF-8")), 
                                           defaultConf);
  }
  private static LineReader makeStream(String str, int bufsz) throws IOException {
    return new LineReader(new ByteArrayInputStream
                                             (str.getBytes("UTF-8")), 
                                           bufsz);
  }

View Full Code Here

                                           bufsz);
  }


  @Test (timeout=5000)
  public void testUTF8() throws Exception {
    LineReader in = makeStream("abcd\u20acbdcd\u20ac");
    Text line = new Text();
    in.readLine(line);
    assertEquals("readLine changed utf8 characters", 
                 "abcd\u20acbdcd\u20ac", line.toString());
    in = makeStream("abc\u200axyz");
    in.readLine(line);
    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
  }

View Full Code Here

  public void testNewLines() throws Exception {
    final String STR = "a\nbb\n\nccc\rdddd\r\r\r\n\r\neeeee";
    final int STRLENBYTES = STR.getBytes().length;
    Text out = new Text();
    for (int bufsz = 1; bufsz < STRLENBYTES+1; ++bufsz) {
      LineReader in = makeStream(STR, bufsz);
      int c = 0;
      c += in.readLine(out); //"a"\n
      assertEquals("line1 length, bufsz:"+bufsz, 1, out.getLength());
      c += in.readLine(out); //"bb"\n
      assertEquals("line2 length, bufsz:"+bufsz, 2, out.getLength());
      c += in.readLine(out); //""\n
      assertEquals("line3 length, bufsz:"+bufsz, 0, out.getLength());
      c += in.readLine(out); //"ccc"\r
      assertEquals("line4 length, bufsz:"+bufsz, 3, out.getLength());
      c += in.readLine(out); //dddd\r
      assertEquals("line5 length, bufsz:"+bufsz, 4, out.getLength());
      c += in.readLine(out); //""\r
      assertEquals("line6 length, bufsz:"+bufsz, 0, out.getLength());
      c += in.readLine(out); //""\r\n
      assertEquals("line7 length, bufsz:"+bufsz, 0, out.getLength());
      c += in.readLine(out); //""\r\n
      assertEquals("line8 length, bufsz:"+bufsz, 0, out.getLength());
      c += in.readLine(out); //"eeeee"EOF
      assertEquals("line9 length, bufsz:"+bufsz, 5, out.getLength());
      assertEquals("end of file, bufsz: "+bufsz, 0, in.readLine(out));
      assertEquals("total bytes, bufsz: "+bufsz, c, STRLENBYTES);
    }
  }

View Full Code Here

  public void testMaxLineLength() throws Exception {
    final String STR = "a\nbb\n\nccc\rdddd\r\neeeee";
    final int STRLENBYTES = STR.getBytes().length;
    Text out = new Text();
    for (int bufsz = 1; bufsz < STRLENBYTES+1; ++bufsz) {
      LineReader in = makeStream(STR, bufsz);
      int c = 0;
      c += in.readLine(out, 1);
      assertEquals("line1 length, bufsz: "+bufsz, 1, out.getLength());
      c += in.readLine(out, 1);
      assertEquals("line2 length, bufsz: "+bufsz, 1, out.getLength());
      c += in.readLine(out, 1);
      assertEquals("line3 length, bufsz: "+bufsz, 0, out.getLength());
      c += in.readLine(out, 3);
      assertEquals("line4 length, bufsz: "+bufsz, 3, out.getLength());
      c += in.readLine(out, 10);
      assertEquals("line5 length, bufsz: "+bufsz, 4, out.getLength());
      c += in.readLine(out, 8);
      assertEquals("line5 length, bufsz: "+bufsz, 5, out.getLength());
      assertEquals("end of file, bufsz: " +bufsz, 0, in.readLine(out));
      assertEquals("total bytes, bufsz: "+bufsz, c, STRLENBYTES);
    }
  }

View Full Code Here

   * @throws Exception
   */
  public static void main(String[] args) throws Exception {
    for(String arg: args) {
      System.out.println("Working on " + arg);
      LineReader reader = makeStream(unquote(arg));
      Text line = new Text();
      int size = reader.readLine(line);
      while (size > 0) {
        System.out.println("Got: " + line.toString());
        size = reader.readLine(line);
      }
      reader.close();
    }
  }

View Full Code Here


    }
  }


  private LineReader makeStream(String str) throws IOException {
    return new LineReader(new ByteArrayInputStream
                                           (str.getBytes("UTF-8")), 
                                           defaultConf);
  }

View Full Code Here

                                           defaultConf);
  }
  
  @Test
  public void testUTF8() throws Exception {
    LineReader in = makeStream("abcd\u20acbdcd\u20ac");
    Text line = new Text();
    in.readLine(line);
    assertEquals("readLine changed utf8 characters", 
                 "abcd\u20acbdcd\u20ac", line.toString());
    in = makeStream("abc\u200axyz");
    in.readLine(line);
    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.util.LineReader

co.nubetech.hiho.dedup.DelimitedLineRecordReader

com.cloudera.iterativereduce.io.HDFSLineParser

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader

com.hadoop.mapred.DeprecatedLzoLineRecordReader

com.hadoop.mapreduce.LzoLineRecordReader

com.twitter.elephantbird.mapreduce.input.LzoBinaryB64LineRecordReader

com.twitter.elephantbird.mapreduce.input.LzoJsonRecordReader

com.twitter.elephantbird.mapreduce.input.LzoLineRecordReader

com.twitter.elephantbird.mapreduce.input.LzoW3CLogRecordReader

edu.umd.cloud9.collection.aquaint2.Aquaint2DocnoMapping

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.