Examples of org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping

Package org.apache.hadoop.hive.hbase.ColumnMappings

Examples of org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping

org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping

    if (sb == null) {
      throw new IllegalArgumentException("StringBuilder cannot be null");
    }


    for (int i = 0; i < columnsMapping.size(); i++) {
      ColumnMapping colMap = columnsMapping.get(i);


      if (colMap.hbaseRowKey) {
        sb.append("key").append(StringUtils.COMMA_STR);
      } else if (colMap.qualifierName == null) {
        // this corresponds to a map<string,?>

View Full Code Here

    for (int i = 0; i < columnsMapping.size(); i++) {
      if (sb.length() > 0) {
        sb.append(":");
      }


      ColumnMapping colMap = columnsMapping.get(i);


      if (colMap.hbaseRowKey) {


        Map<String, String> compositeKeyParts = getCompositeKeyParts(tbl);
        StringBuilder keyStruct = new StringBuilder();

View Full Code Here

  /**
   * Create the {@link LazyObjectBase lazy field}
   * */
  public static LazyObjectBase createLazyField(ColumnMapping[] columnMappings, int fieldID,
      ObjectInspector inspector) {
    ColumnMapping colMap = columnMappings[fieldID];
    if (colMap.getQualifierName() == null && !colMap.isHbaseRowKey()) {
      // a column family
      return new LazyHBaseCellMap((LazyMapObjectInspector) inspector);
    }
    return LazyFactory.createLazyObject(inspector, colMap.getBinaryStorage().get(0));
  }

View Full Code Here

      if (idxFirst < 0 || !(idxFirst == idxLast)) {
        throw new SerDeException("Error: the HBase columns mapping contains a badly formed " +
            "column family, column qualifier specification.");
      }


      ColumnMapping columnMapping = new ColumnMapping();


      if (colInfo.equals(HBASE_KEY_COL)) {
        rowKeyIndex = i;
        columnMapping.familyName = colInfo;
        columnMapping.familyNameBytes = Bytes.toBytes(colInfo);
        columnMapping.qualifierName = null;
        columnMapping.qualifierNameBytes = null;
        columnMapping.hbaseRowKey = true;
      } else if (colInfo.equals(HBASE_TIMESTAMP_COL)) {
        timestampIndex = i;
        columnMapping.familyName = colInfo;
        columnMapping.familyNameBytes = Bytes.toBytes(colInfo);
        columnMapping.qualifierName = null;
        columnMapping.qualifierNameBytes = null;
        columnMapping.hbaseTimestamp = true;
      } else {
        String [] parts = colInfo.split(":");
        assert(parts.length > 0 && parts.length <= 2);
        columnMapping.familyName = parts[0];
        columnMapping.familyNameBytes = Bytes.toBytes(parts[0]);
        columnMapping.hbaseRowKey = false;
        columnMapping.hbaseTimestamp = false;


        if (parts.length == 2) {


          if (doColumnRegexMatching && parts[1].endsWith(".*")) {
            // we have a prefix with a wildcard
            columnMapping.qualifierPrefix = parts[1].substring(0, parts[1].length() - 2);
            columnMapping.qualifierPrefixBytes = Bytes.toBytes(columnMapping.qualifierPrefix);
            // we weren't provided any actual qualifier name. Set these to
            // null.
            columnMapping.qualifierName = null;
            columnMapping.qualifierNameBytes = null;
          } else {
            // set the regular provided qualifier names
            columnMapping.qualifierName = parts[1];
            columnMapping.qualifierNameBytes = Bytes.toBytes(parts[1]);
          }
        } else {
          columnMapping.qualifierName = null;
          columnMapping.qualifierNameBytes = null;
        }
      }


      columnMapping.mappingSpec = mappingSpec;


      columnsMapping.add(columnMapping);
    }


    if (rowKeyIndex == -1) {
      rowKeyIndex = 0;
      ColumnMapping columnMapping = new ColumnMapping();
      columnMapping.familyName = HBaseSerDe.HBASE_KEY_COL;
      columnMapping.familyNameBytes = Bytes.toBytes(HBaseSerDe.HBASE_KEY_COL);
      columnMapping.qualifierName = null;
      columnMapping.qualifierNameBytes = null;
      columnMapping.hbaseRowKey = true;

View Full Code Here

    LazyObjectBase[] fields = getFields();
    boolean [] fieldsInited = getFieldInited();


    if (!fieldsInited[fieldID]) {
      ByteArrayRef ref = null;
      ColumnMapping colMap = columnsMapping[fieldID];


      if (colMap.hbaseRowKey) {
        ref = new ByteArrayRef();
        ref.setData(result.getRow());
      } else if (colMap.hbaseTimestamp) {

View Full Code Here

    List<String> addedFamilies = new ArrayList<String>();


    if (!readAllColumns) {
      ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping();
      for (int i : readColIDs) {
        ColumnMapping colMap = columnsMapping[i];
        if (colMap.hbaseRowKey || colMap.hbaseTimestamp) {
          continue;
        }


        if (colMap.qualifierName == null) {

View Full Code Here

      throw new IOException(e);
    }


    int iKey = columnMappings.getKeyIndex();
    int iTimestamp = columnMappings.getTimestampIndex();
    ColumnMapping keyMapping = columnMappings.getKeyMapping();


    // Take filter pushdown into account while calculating splits; this
    // allows us to prune off regions immediately.  Note that although
    // the Javadoc for the superclass getSplits says that it returns one
    // split per region, the implementation actually takes the scan

View Full Code Here


  public static DecomposedPredicate decomposePredicate(
      JobConf jobConf,
      HBaseSerDe hBaseSerDe,
      ExprNodeDesc predicate) {
    ColumnMapping keyMapping = hBaseSerDe.getHBaseSerdeParam().getKeyColumnMapping();
    ColumnMapping tsMapping = hBaseSerDe.getHBaseSerdeParam().getTimestampColumnMapping();
    IndexPredicateAnalyzer analyzer = HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(
        keyMapping.columnName, keyMapping.isComparable(),
        tsMapping == null ? null : tsMapping.columnName);
    List<IndexSearchCondition> conditions = new ArrayList<IndexSearchCondition>();
    ExprNodeGenericFuncDesc residualPredicate =

View Full Code Here

      fail(e.toString());
    }


    ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping();
    for (int i = 0; i < columnsMapping.length; i++) {
      ColumnMapping colMap = columnsMapping[i];


      if (i == 0 || i == 7) {
        colMap.binaryStorage.add(false);
      } else {
        colMap.binaryStorage.add(true);
      }
    }


    ObjectInspector oi =
      LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos,
          new byte [] {' ', ':', '='}, nullSequence, false, false, (byte) 0);


    LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings);


    byte [] rowKey = "row-key".getBytes();
    List<KeyValue> kvs = new ArrayList<KeyValue>();
    byte [] value;


    for (int i = 1; i < columnsMapping.length; i++) {


      switch (i) {


      case 1:
        value = Bytes.toBytes(1);
        break;


      case 2:
        value = new byte[]{(byte)1};
        break;


      case 3:
        value = Bytes.toBytes((short) 1);
        break;


      case 4:
        value = Bytes.toBytes((long) 1);
        break;


      case 5:
        value = Bytes.toBytes((float) 1.0F);
        break;


      case 6:
        value = Bytes.toBytes((double) 1.0);
        break;


      case 7:
        value = "Hadoop, Hive, with HBase storage handler.".getBytes();
        break;


      case 8:
        value = Bytes.toBytes(true);
        break;


      default:
        throw new RuntimeException("Not expected: " + i);
      }


      ColumnMapping colMap = columnsMapping[i];
      kvs.add(new KeyValue(rowKey, colMap.familyNameBytes, colMap.qualifierNameBytes, value));
    }


    Collections.sort(kvs, KeyValue.COMPARATOR);
    Result result = new Result(kvs);

View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping

org.apache.hadoop.hive.hbase.HBaseSerDe

org.apache.hadoop.hive.hbase.HBaseSerDeHelper

org.apache.hadoop.hive.hbase.HBaseStorageHandler

org.apache.hadoop.hive.hbase.HiveHBaseInputFormatUtil

org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat

org.apache.hadoop.hive.hbase.LazyHBaseRow

org.apache.hadoop.hive.hbase.TestLazyHBaseObject

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.