Examples of com.dyuproject.protostuff.CompareInputsTest$Deserializer

org.apache.hadoop.hive.serde2.Deserializer
Benchmark to compare the deserialization speed of 2 types. CodedInput and ByteArrayInput. @author David Yu @created Jun 22, 2010

    JobConf jobConf = new JobConf(owi.getParseContext().getConf());
    Utilities.setColumnNameList(jobConf, tableScanOp);
    Utilities.copyTableJobPropertiesToConf(
      Utilities.getTableDesc(tbl),
      jobConf);
    Deserializer deserializer = tbl.getDeserializer();
    HiveStoragePredicateHandler.DecomposedPredicate decomposed =
      predicateHandler.decomposePredicate(
        jobConf,
        deserializer,
        originalPredicate);

View Full Code Here


  public ArrayList<StructField> getFields() {


    ArrayList<StructField> fields = new ArrayList<StructField>();
    try {
      Deserializer decoder = getDeserializer();


      // Expand out all the columns of the table
      StructObjectInspector structObjectInspector = (StructObjectInspector) decoder
          .getObjectInspector();
      List<? extends StructField> fld_lst = structObjectInspector
          .getAllStructFieldRefs();
      for (StructField field : fld_lst) {
        fields.add(field);

View Full Code Here


  public Vector<StructField> getFields() {


    Vector<StructField> fields = new Vector<StructField> ();
    try {
      Deserializer decoder = getDeserializer();


      // Expand out all the columns of the table
      StructObjectInspector structObjectInspector = (StructObjectInspector)decoder.getObjectInspector();
      List<? extends StructField> fld_lst = structObjectInspector.getAllStructFieldRefs();
      for(StructField field: fld_lst) {
        fields.add(field);
      }
    } catch (SerDeException e) {

View Full Code Here

  /**
   * Check if the given serde is valid.
   */
  private void validateSerDe(String serdeName) throws HiveException {
    try {
      Deserializer d = SerDeUtils.lookupDeserializer(serdeName);
      if (d != null) {
        LOG.debug("Found class for " + serdeName);
      }
    } catch (SerDeException e) {
      throw new HiveException("Cannot validate serde: " + serdeName, e);

View Full Code Here


    String tableName = String.valueOf(tblProps.getProperty("name"));
    String partName = String.valueOf(partSpec);
    // HiveConf.setVar(hconf, HiveConf.ConfVars.HIVETABLENAME, tableName);
    // HiveConf.setVar(hconf, HiveConf.ConfVars.HIVEPARTITIONNAME, partName);
    Deserializer deserializer = (Deserializer) sdclass.newInstance();
    deserializer.initialize(hconf, tblProps);
    StructObjectInspector rawRowObjectInspector = (StructObjectInspector) deserializer
        .getObjectInspector();


    MapOpCtx opCtx = null;
    // Next check if this table has partitions and if so
    // get the list of partition names as well as allocate

View Full Code Here

      final TableDesc tblDesc) throws HiveException {
    tableDesc = tblDesc;
    properties = part.getSchemaFromTableSchema(tblDesc.getProperties()); // each partition maintains a large properties
    partSpec = part.getSpec();
    // deserializerClass = part.getDeserializer(properties).getClass();
    Deserializer deserializer;
    try {
      deserializer = SerDeUtils.lookupDeserializer(
          properties.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB));
    } catch (SerDeException e) {
      throw new HiveException(e);
    }
    deserializerClass = deserializer.getClass();
    inputFileFormatClass = part.getInputFormatClass();
    outputFileFormatClass = part.getOutputFormatClass();
    serdeClassName = properties.getProperty(
        org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB);
  }

View Full Code Here


  /**
   * Return a deserializer object corresponding to the tableDesc.
   */
  public Deserializer getDeserializer() throws Exception {
    Deserializer de = deserializerClass.newInstance();
    de.initialize(null, properties);
    return de;
  }

View Full Code Here

            tbl.getSd().getSerdeInfo().getSerializationLib());
        if (!getColsFromSerDe) {
          ret = tbl.getSd().getCols();
        } else {
          try {
            Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl);
            ret = MetaStoreUtils.getFieldsFromDeserializer(tableName, s);
          } catch (SerDeException e) {
            StringUtils.stringifyException(e);
            throw new MetaException(e.getMessage());
          }

View Full Code Here

  static public Deserializer getDeserializer(Configuration conf,
      Properties schema) throws MetaException {
    String lib = schema
        .getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB);
    try {
      Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
      (deserializer).initialize(conf, schema);
      return deserializer;
    } catch (Exception e) {
      log.error(e, "error in initSerDe: %s %s", e.getClass().getName(), e.getMessage());
      MetaStoreUtils.printStackTrace(e);

View Full Code Here

    String lib = table.getSd().getSerdeInfo().getSerializationLib();
    if (lib == null) {
      return null;
    }
    try {
      Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
      deserializer.initialize(conf, MetaStoreUtils.getTableMetadata(table));
      return deserializer;
    } catch (RuntimeException e) {
      throw e;
    } catch (Exception e) {
      log.error(e, "error in initSerDe: %s %s",  e.getClass().getName(), e.getMessage());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.dyuproject.protostuff.CompareInputsTest$Deserializer

com.facebook.giraph.hive.impl.input.InputSplitData

com.facebook.giraph.hive.input.HiveApiInputFormat

com.facebook.giraph.hive.input.InputSplitData

com.facebook.hiveio.input.HiveApiInputFormat

com.facebook.hiveio.input.InputSplitData

com.facebook.presto.hive.DwrfRecordCursorProvider

com.facebook.presto.hive.HiveUtil

com.facebook.presto.hive.OrcRecordCursorProvider

com.facebook.presto.hive.ParquetRecordCursorProvider

com.facebook.presto.jdbc.internal.jackson.core.io.SerializedString

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.