Examples of TCTLSeparatedProtocol

org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol
An implementation of the Thrift Protocol for ctl separated records. This is not thrift compliant in that it doesn't write out field ids so things cannot actually be versioned.

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    prot.readStructEnd();
  }


  public void testQuotedWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(4096);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();
    schema.setProperty(serdeConstants.QUOTE_CHAR, "\"");
    schema.setProperty(serdeConstants.FIELD_DELIM, ",");
    prot.initialize(new Configuration(), schema);


    String testStr = "\"hello, world!\"";


    prot.writeStructBegin(new TStruct());


    prot.writeFieldBegin(new TField());
    prot.writeString(testStr);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();


    prot.writeStructEnd();
    prot.writeString("\n");


    trans.flush();


    byte b[] = new byte[4096];
    int len = trans.read(b, 0, b.length);


    trans = new TMemoryBuffer(4096);
    trans.write(b, 0, len);
    prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), schema);


    prot.readStructBegin();
    prot.readFieldBegin();
    final String firstRead = prot.readString();
    prot.readFieldEnd();


    testStr = testStr.replace("\"", "");


    assertEquals(testStr, firstRead);


    // the 2 element list
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();


    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();


    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();


    prot.readStructEnd();
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol


    TMemoryBuffer trans = new TMemoryBuffer(4096);
    trans.write(sample.getBytes(), 0, sample.getBytes().length);
    trans.flush();


    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();


    // this is a hacky way of doing the quotes since it will match any 2 of
    // these, so
    // "[ hello this is something to split [" would be considered to be quoted.
    schema.setProperty(serdeConstants.QUOTE_CHAR, "(\"|\\[|\\])");


    schema.setProperty(serdeConstants.FIELD_DELIM, " ");
    schema.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "-");
    prot.initialize(new Configuration(), schema);


    prot.readStructBegin();


    // ip address
    prot.readFieldBegin();
    final String ip = prot.readString();
    prot.readFieldEnd();


    assertEquals("127.0.0.1", ip);


    // identd
    prot.readFieldBegin();
    final String identd = prot.readString();
    prot.readFieldEnd();


    assertNull(identd);


    // user
    prot.readFieldBegin();
    final String user = prot.readString();
    prot.readFieldEnd();


    assertEquals("frank", user);


    // finishTime
    prot.readFieldBegin();
    final String finishTime = prot.readString();
    prot.readFieldEnd();


    assertEquals("10/Oct/2000:13:55:36 -0700", finishTime);


    // requestLine
    prot.readFieldBegin();
    final String requestLine = prot.readString();
    prot.readFieldEnd();


    assertEquals("GET /apache_pb.gif HTTP/1.0", requestLine);


    // returncode
    prot.readFieldBegin();
    final int returnCode = prot.readI32();
    prot.readFieldEnd();


    assertEquals(200, returnCode);


    // return size
    prot.readFieldBegin();
    final int returnSize = prot.readI32();
    prot.readFieldEnd();


    assertEquals(2326, returnSize);


    prot.readStructEnd();
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    prot.readStructEnd();
  }


  public void testNulls() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 10);
    prot.initialize(new Configuration(), new Properties());


    prot.writeStructBegin(new TStruct());


    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeI32(100);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeMapBegin(new TMap());
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("key2");
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("val3");
    prot.writeMapEnd();
    prot.writeFieldEnd();


    prot.writeStructEnd();


    byte b[] = new byte[3 * 1024];
    int len = trans.read(b, 0, b.length);
    String written = new String(b, 0, len);


    String testRef = "\\N\\N100\\N\\N\\Nkey2\\N\\Nval3";


    assertTrue(testRef.equals(written));


    trans = new TMemoryBuffer(1023);
    trans.write(b, 0, len);


    prot = new TCTLSeparatedProtocol(trans, 3);
    prot.initialize(new Configuration(), new Properties());


    prot.readStructBegin();


    prot.readFieldBegin();
    String ret = prot.readString();
    prot.readFieldEnd();


    assertNull(ret);


    prot.readFieldBegin();
    ret = prot.readString();
    prot.readFieldEnd();


    assertNull(ret);


    prot.readFieldBegin();
    int ret1 = prot.readI32();
    prot.readFieldEnd();


    assertTrue(ret1 == 100);


    prot.readFieldBegin();
    ret1 = prot.readI32();
    prot.readFieldEnd();


    prot.readFieldBegin();
    TMap map = prot.readMapBegin();


    assertTrue(map.size == 3);


    assertNull(prot.readString());
    assertNull(prot.readString());


    assertTrue(prot.readString().equals("key2"));
    assertNull(prot.readString());


    assertNull(prot.readString());
    assertTrue(prot.readString().equals("val3"));


    prot.readMapEnd();
    prot.readFieldEnd();


    assertTrue(ret1 == 0);
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol


    assertTrue(ret1 == 0);
  }


  public void testShouldThrowRunTimeExceptionIfUnableToInitializeTokenizer() throws Exception {
    TCTLSeparatedProtocol separatedProtocol = new TCTLSeparatedProtocol(new TTransport() {
      @Override
      public void close() {
      }


      @Override
      public boolean isOpen() {
        return false;
      }


      @Override
      public void open() throws TTransportException {
      }


      @Override
      public int read(byte[] buf, int off, int len) throws TTransportException {
        throw new TTransportException();
      }


      @Override
      public void write(byte[] buf, int off, int len) throws TTransportException {
      }
    });
    separatedProtocol.initialize(null, new Properties());
    try {
      separatedProtocol.readStructBegin();
      fail("Runtime Exception is expected if the intialization of tokenizer failed.");
    } catch (Exception e) {
      assertTrue(e.getCause() instanceof TTransportException);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

      schema.setProperty(serdeConstants.MAPKEY_DELIM, "4");


      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);


      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol) serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));


      ObjectInspector oi = serde.getObjectInspector();


      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

      schema.setProperty(serdeConstants.MAPKEY_DELIM, "4");


      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);


      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol) serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));


      ObjectInspector oi = serde.getObjectInspector();


      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    prot.readStructEnd();
  }


  public void testQuotedWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(4096);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();
    schema.setProperty(Constants.QUOTE_CHAR, "\"");
    schema.setProperty(Constants.FIELD_DELIM, ",");
    prot.initialize(new Configuration(), schema);


    String testStr = "\"hello, world!\"";


    prot.writeStructBegin(new TStruct());


    prot.writeFieldBegin(new TField());
    prot.writeString(testStr);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();


    prot.writeStructEnd();
    prot.writeString("\n");


    trans.flush();


    byte b[] = new byte[4096];
    int len = trans.read(b, 0, b.length);


    trans = new TMemoryBuffer(4096);
    trans.write(b, 0, len);
    prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), schema);


    prot.readStructBegin();
    prot.readFieldBegin();
    final String firstRead = prot.readString();
    prot.readFieldEnd();


    testStr = testStr.replace("\"", "");


    assertEquals(testStr, firstRead);


    // the 2 element list
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();


    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();


    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();


    prot.readStructEnd();
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol


    TMemoryBuffer trans = new TMemoryBuffer(4096);
    trans.write(sample.getBytes(), 0, sample.getBytes().length);
    trans.flush();


    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();


    // this is a hacky way of doing the quotes since it will match any 2 of
    // these, so
    // "[ hello this is something to split [" would be considered to be quoted.
    schema.setProperty(Constants.QUOTE_CHAR, "(\"|\\[|\\])");


    schema.setProperty(Constants.FIELD_DELIM, " ");
    schema.setProperty(Constants.SERIALIZATION_NULL_FORMAT, "-");
    prot.initialize(new Configuration(), schema);


    prot.readStructBegin();


    // ip address
    prot.readFieldBegin();
    final String ip = prot.readString();
    prot.readFieldEnd();


    assertEquals("127.0.0.1", ip);


    // identd
    prot.readFieldBegin();
    final String identd = prot.readString();
    prot.readFieldEnd();


    assertNull(identd);


    // user
    prot.readFieldBegin();
    final String user = prot.readString();
    prot.readFieldEnd();


    assertEquals("frank", user);


    // finishTime
    prot.readFieldBegin();
    final String finishTime = prot.readString();
    prot.readFieldEnd();


    assertEquals("10/Oct/2000:13:55:36 -0700", finishTime);


    // requestLine
    prot.readFieldBegin();
    final String requestLine = prot.readString();
    prot.readFieldEnd();


    assertEquals("GET /apache_pb.gif HTTP/1.0", requestLine);


    // returncode
    prot.readFieldBegin();
    final int returnCode = prot.readI32();
    prot.readFieldEnd();


    assertEquals(200, returnCode);


    // return size
    prot.readFieldBegin();
    final int returnSize = prot.readI32();
    prot.readFieldEnd();


    assertEquals(2326, returnSize);


    prot.readStructEnd();
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    prot.readStructEnd();
  }


  public void testNulls() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 10);
    prot.initialize(new Configuration(), new Properties());


    prot.writeStructBegin(new TStruct());


    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeI32(100);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();


    prot.writeFieldBegin(new TField());
    prot.writeMapBegin(new TMap());
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("key2");
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("val3");
    prot.writeMapEnd();
    prot.writeFieldEnd();


    prot.writeStructEnd();


    byte b[] = new byte[3 * 1024];
    int len = trans.read(b, 0, b.length);
    String written = new String(b, 0, len);


    String testRef = "\\N\\N100\\N\\N\\Nkey2\\N\\Nval3";


    assertTrue(testRef.equals(written));


    trans = new TMemoryBuffer(1023);
    trans.write(b, 0, len);


    prot = new TCTLSeparatedProtocol(trans, 3);
    prot.initialize(new Configuration(), new Properties());


    prot.readStructBegin();


    prot.readFieldBegin();
    String ret = prot.readString();
    prot.readFieldEnd();


    assertNull(ret);


    prot.readFieldBegin();
    ret = prot.readString();
    prot.readFieldEnd();


    assertNull(ret);


    prot.readFieldBegin();
    int ret1 = prot.readI32();
    prot.readFieldEnd();


    assertTrue(ret1 == 100);


    prot.readFieldBegin();
    ret1 = prot.readI32();
    prot.readFieldEnd();


    prot.readFieldBegin();
    TMap map = prot.readMapBegin();


    assertTrue(map.size == 3);


    assertNull(prot.readString());
    assertNull(prot.readString());


    assertTrue(prot.readString().equals("key2"));
    assertNull(prot.readString());


    assertNull(prot.readString());
    assertTrue(prot.readString().equals("val3"));


    prot.readMapEnd();
    prot.readFieldEnd();


    assertTrue(ret1 == 0);
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    trans.write(value2.getBytes(), 0, value2.getBytes().length);


    trans.flush();


    // use 3 as the row buffer size to force lots of re-buffering.
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), new Properties());


    prot.readStructBegin();


    prot.readFieldBegin();
    String hello = prot.readString();
    prot.readFieldEnd();


    assertTrue(hello.equals(foo));


    prot.readFieldBegin();
    assertTrue(prot.readString().equals(""));
    prot.readFieldEnd();


    prot.readFieldBegin();
    assertTrue(prot.readString().equals(bar));
    prot.readFieldEnd();


    prot.readFieldBegin();
    TMap mapHeader = prot.readMapBegin();
    assertTrue(mapHeader.size == 2);


    assertTrue(prot.readI32() == 22);
    assertTrue(prot.readString().equals(value));
    assertTrue(prot.readI32() == 24);
    assertTrue(prot.readString().equals(value2));
    prot.readMapEnd();
    prot.readFieldEnd();


    prot.readFieldBegin();
    hello = prot.readString();
    prot.readFieldEnd();
    assertNull(hello);


    prot.readStructEnd();
  }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.