Examples of TsvParser

com.asakusafw.runtime.io.TsvParser

TSVファイルを解析してレコードを読み出す。

次のように利用する。

 Reader reader = ...; SomeModel model = new SomeModel(); try { TsvParser parser = new TsvParser(reader); while (parser.hasNext()) { parser.fill(model.getHogeOption()); parser.fill(model.getFooOption()); parser.fill(model.getBarOption()); performModelAction(model); } } finally { reader.close(); }

特に指定がない限り、このクラスのメソッドの引数に {@code null}を指定した場合には {@link NullPointerException}がスローされる。

org.apache.hadoop.hbase.index.mapreduce.IndexImportTsv.TsvParser
org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    parser.parse(line, line.length);
  }


  @Test(expected = BadTsvLineException.class)
  public void testTsvParserInvalidTimestamp() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t");
    assertEquals(1, parser.getTimestampKeyColumnIndex());
    byte[] line = Bytes.toBytes("rowkey\ttimestamp\tval_a");
    ParsedLine parsed = parser.parse(line, line.length);
    assertEquals(-1, parsed.getTimestamp(-1));
    checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
  }


  @Test(expected = BadTsvLineException.class)
  public void testTsvParserNoTimestampValue() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY", "\t");
    assertEquals(2, parser.getTimestampKeyColumnIndex());
    byte[] line = Bytes.toBytes("rowkey\tval_a");
    parser.parse(line, line.length);
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    parser.parse(line, line.length);
  }


  @Test
  public void testTsvParserParseRowKey() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY", "\t");
    assertEquals(0, parser.getRowKeyColumnIndex());
    byte[] line = Bytes.toBytes("rowkey\tval_a\t1234");
    Pair<Integer, Integer> rowKeyOffsets = parser.parseRowKey(line, line.length);
    assertEquals(0, rowKeyOffsets.getFirst().intValue());
    assertEquals(5, rowKeyOffsets.getSecond().intValue());
    try {
      line = Bytes.toBytes("\t\tval_a\t1234");
      parser.parseRowKey(line, line.length);
      fail("Should get BadTsvLineException on empty rowkey.");
    } catch (BadTsvLineException b) {


    }
    parser = new TsvParser("col_a,HBASE_ROW_KEY,HBASE_TS_KEY", "\t");
    assertEquals(1, parser.getRowKeyColumnIndex());
    line = Bytes.toBytes("val_a\trowkey\t1234");
    rowKeyOffsets = parser.parseRowKey(line, line.length);
    assertEquals(6, rowKeyOffsets.getFirst().intValue());
    assertEquals(11, rowKeyOffsets.getSecond().intValue());
    try {
      line = Bytes.toBytes("val_a");
      rowKeyOffsets = parser.parseRowKey(line, line.length);
      fail("Should get BadTsvLineException when number of columns less than rowkey position.");
    } catch (BadTsvLineException b) {


    }
    parser = new TsvParser("col_a,HBASE_TS_KEY,HBASE_ROW_KEY", "\t");
    assertEquals(2, parser.getRowKeyColumnIndex());
    line = Bytes.toBytes("val_a\t1234\trowkey");
    rowKeyOffsets = parser.parseRowKey(line, line.length);
    assertEquals(11, rowKeyOffsets.getFirst().intValue());
    assertEquals(16, rowKeyOffsets.getSecond().intValue());
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    assertEquals(16, rowKeyOffsets.getSecond().intValue());
  }


  @Test
  public void testTsvParseAttributesKey() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a,HBASE_TS_KEY,HBASE_ATTRIBUTES_KEY", "\t");
    assertEquals(0, parser.getRowKeyColumnIndex());
    byte[] line = Bytes.toBytes("rowkey\tval_a\t1234\tkey=>value");
    ParsedLine parse = parser.parse(line, line.length);
    assertEquals(18, parse.getAttributeKeyOffset());
    assertEquals(3, parser.getAttributesKeyColumnIndex());
    String attributes[] = parse.getIndividualAttributes();
    assertEquals(attributes[0], "key=>value");
    try {
      line = Bytes.toBytes("rowkey\tval_a\t1234");
      parser.parse(line, line.length);
      fail("Should get BadTsvLineException on empty rowkey.");
    } catch (BadTsvLineException b) {


    }
    parser = new TsvParser("HBASE_ATTRIBUTES_KEY,col_a,HBASE_ROW_KEY,HBASE_TS_KEY", "\t");
    assertEquals(2, parser.getRowKeyColumnIndex());
    line = Bytes.toBytes("key=>value\tval_a\trowkey\t1234");
    parse = parser.parse(line, line.length);
    assertEquals(0, parse.getAttributeKeyOffset());
    assertEquals(0, parser.getAttributesKeyColumnIndex());
    attributes = parse.getIndividualAttributes();
    assertEquals(attributes[0], "key=>value");
    try {
      line = Bytes.toBytes("val_a");
      ParsedLine parse2 = parser.parse(line, line.length);
      fail("Should get BadTsvLineException when number of columns less than rowkey position.");
    } catch (BadTsvLineException b) {


    }
    parser = new TsvParser("col_a,HBASE_ATTRIBUTES_KEY,HBASE_TS_KEY,HBASE_ROW_KEY", "\t");
    assertEquals(3, parser.getRowKeyColumnIndex());
    line = Bytes.toBytes("val_a\tkey0=>value0,key1=>value1,key2=>value2\t1234\trowkey");
    parse = parser.parse(line, line.length);
    assertEquals(1, parser.getAttributesKeyColumnIndex());
    assertEquals(6, parse.getAttributeKeyOffset());
    String[] attr = parse.getIndividualAttributes();
    int i = 0;
    for(String str :  attr) {
      assertEquals(("key"+i+"=>"+"value"+i), str );

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    }
  }


  @Test
  public void testTsvParserWithCellVisibilityCol() throws BadTsvLineException {
    TsvParser parser = new TsvParser(
        "HBASE_ROW_KEY,col_a,HBASE_TS_KEY,HBASE_ATTRIBUTES_KEY,HBASE_CELL_VISIBILITY", "\t");
    assertEquals(0, parser.getRowKeyColumnIndex());
    assertEquals(4, parser.getCellVisibilityColumnIndex());
    byte[] line = Bytes.toBytes("rowkey\tval_a\t1234\tkey=>value\tPRIVATE&SECRET");
    ParsedLine parse = parser.parse(line, line.length);
    assertEquals(18, parse.getAttributeKeyOffset());
    assertEquals(3, parser.getAttributesKeyColumnIndex());
    String attributes[] = parse.getIndividualAttributes();
    assertEquals(attributes[0], "key=>value");
    assertEquals(29, parse.getCellVisibilityColumnOffset());
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

public class TestImportTsv {
  private static final Log LOG = LogFactory.getLog(TestImportTsv.class);


  @Test
  public void testTsvParserSpecParsing() {
    TsvParser parser;


    parser = new TsvParser("HBASE_ROW_KEY", "\t");
    assertNull(parser.getFamily(0));
    assertNull(parser.getQualifier(0));
    assertEquals(0, parser.getRowKeyColumnIndex());
    assertFalse(parser.hasTimestamp());


    parser = new TsvParser("HBASE_ROW_KEY,col1:scol1", "\t");
    assertNull(parser.getFamily(0));
    assertNull(parser.getQualifier(0));
    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1));
    assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1));
    assertEquals(0, parser.getRowKeyColumnIndex());
    assertFalse(parser.hasTimestamp());


    parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,col1:scol2", "\t");
    assertNull(parser.getFamily(0));
    assertNull(parser.getQualifier(0));
    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1));
    assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1));
    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(2));
    assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(2));
    assertEquals(0, parser.getRowKeyColumnIndex());
    assertFalse(parser.hasTimestamp());
    
    parser = new TsvParser("HBASE_ROW_KEY,col1:scol1,HBASE_TS_KEY,col1:scol2",
        "\t");
    assertNull(parser.getFamily(0));
    assertNull(parser.getQualifier(0));
    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(1));
    assertBytesEquals(Bytes.toBytes("scol1"), parser.getQualifier(1));
    assertBytesEquals(Bytes.toBytes("col1"), parser.getFamily(3));
    assertBytesEquals(Bytes.toBytes("scol2"), parser.getQualifier(3));
    assertEquals(0, parser.getRowKeyColumnIndex());
    assertTrue(parser.hasTimestamp());
    assertEquals(2, parser.getTimestampKeyColumnIndex());
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    assertEquals(2, parser.getTimestampKeyColumnIndex());
  }


  @Test
  public void testTsvParser() throws BadTsvLineException {
    TsvParser parser = new TsvParser("col_a,col_b:qual,HBASE_ROW_KEY,col_d", "\t");
    assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(0));
    assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(0));
    assertBytesEquals(Bytes.toBytes("col_b"), parser.getFamily(1));
    assertBytesEquals(Bytes.toBytes("qual"), parser.getQualifier(1));
    assertNull(parser.getFamily(2));
    assertNull(parser.getQualifier(2));
    assertEquals(2, parser.getRowKeyColumnIndex());
    
    assertEquals(TsvParser.DEFAULT_TIMESTAMP_COLUMN_INDEX, parser
        .getTimestampKeyColumnIndex());
    
    byte[] line = Bytes.toBytes("val_a\tval_b\tval_c\tval_d");
    ParsedLine parsed = parser.parse(line, line.length);
    checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

  }
  
  
  @Test
  public void testTsvParserWithTimestamp() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,HBASE_TS_KEY,col_a,", "\t");
    assertNull(parser.getFamily(0));
    assertNull(parser.getQualifier(0));
    assertNull(parser.getFamily(1));
    assertNull(parser.getQualifier(1));
    assertBytesEquals(Bytes.toBytes("col_a"), parser.getFamily(2));
    assertBytesEquals(HConstants.EMPTY_BYTE_ARRAY, parser.getQualifier(2));
    assertEquals(0, parser.getRowKeyColumnIndex());
    assertEquals(1, parser.getTimestampKeyColumnIndex());


    byte[] line = Bytes.toBytes("rowkey\t1234\tval_a");
    ParsedLine parsed = parser.parse(line, line.length);
    assertEquals(1234l, parsed.getTimestamp(-1));
    checkParsing(parsed, Splitter.on("\t").split(Bytes.toString(line)));
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

  /**
   * Test cases that throw BadTsvLineException
   */
  @Test(expected=BadTsvLineException.class)
  public void testTsvParserBadTsvLineExcessiveColumns() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t");
    byte[] line = Bytes.toBytes("val_a\tval_b\tval_c");
    parser.parse(line, line.length);
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser

    parser.parse(line, line.length);
  }


  @Test(expected=BadTsvLineException.class)
  public void testTsvParserBadTsvLineZeroColumn() throws BadTsvLineException {
    TsvParser parser = new TsvParser("HBASE_ROW_KEY,col_a", "\t");
    byte[] line = Bytes.toBytes("");
    parser.parse(line, line.length);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.