Package org.apache.hadoop.hive.serde2.objectinspector

Examples of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector


    // This field is not a null.
    buffer.write((byte) 1, invert);

    switch (oi.getCategory()) {
    case PRIMITIVE: {
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
      switch (poi.getPrimitiveCategory()) {
      case VOID: {
        return;
      }
      case BOOLEAN: {
        boolean v = ((BooleanObjectInspector) poi).get(o);
        buffer.write((byte) (v ? 2 : 1), invert);
        return;
      }
      case BYTE: {
        ByteObjectInspector boi = (ByteObjectInspector) poi;
        byte v = boi.get(o);
        buffer.write((byte) (v ^ 0x80), invert);
        return;
      }
      case SHORT: {
        ShortObjectInspector spoi = (ShortObjectInspector) poi;
        short v = spoi.get(o);
        buffer.write((byte) ((v >> 8) ^ 0x80), invert);
        buffer.write((byte) v, invert);
        return;
      }
      case INT: {
        IntObjectInspector ioi = (IntObjectInspector) poi;
        int v = ioi.get(o);
        serializeInt(buffer, v, invert);
        return;
      }
      case LONG: {
        LongObjectInspector loi = (LongObjectInspector) poi;
        long v = loi.get(o);
        buffer.write((byte) ((v >> 56) ^ 0x80), invert);
        buffer.write((byte) (v >> 48), invert);
        buffer.write((byte) (v >> 40), invert);
        buffer.write((byte) (v >> 32), invert);
        buffer.write((byte) (v >> 24), invert);
        buffer.write((byte) (v >> 16), invert);
        buffer.write((byte) (v >> 8), invert);
        buffer.write((byte) v, invert);
        return;
      }
      case FLOAT: {
        FloatObjectInspector foi = (FloatObjectInspector) poi;
        int v = Float.floatToIntBits(foi.get(o));
        if ((v & (1 << 31)) != 0) {
          // negative number, flip all bits
          v = ~v;
        } else {
          // positive number, flip the first bit
          v = v ^ (1 << 31);
        }
        buffer.write((byte) (v >> 24), invert);
        buffer.write((byte) (v >> 16), invert);
        buffer.write((byte) (v >> 8), invert);
        buffer.write((byte) v, invert);
        return;
      }
      case DOUBLE: {
        DoubleObjectInspector doi = (DoubleObjectInspector) poi;
        long v = Double.doubleToLongBits(doi.get(o));
        if ((v & (1L << 63)) != 0) {
          // negative number, flip all bits
          v = ~v;
        } else {
          // positive number, flip the first bit
          v = v ^ (1L << 63);
        }
        buffer.write((byte) (v >> 56), invert);
        buffer.write((byte) (v >> 48), invert);
        buffer.write((byte) (v >> 40), invert);
        buffer.write((byte) (v >> 32), invert);
        buffer.write((byte) (v >> 24), invert);
        buffer.write((byte) (v >> 16), invert);
        buffer.write((byte) (v >> 8), invert);
        buffer.write((byte) v, invert);
        return;
      }
      case STRING: {
        StringObjectInspector soi = (StringObjectInspector) poi;
        Text t = soi.getPrimitiveWritableObject(o);
        serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
        return;
      }

      case CHAR: {
        HiveCharObjectInspector hcoi = (HiveCharObjectInspector) poi;
        HiveCharWritable hc = hcoi.getPrimitiveWritableObject(o);
        // Trailing space should ignored for char comparisons.
        // So write stripped values for this SerDe.
        Text t = hc.getStrippedValue();
        serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
        return;
      }
      case VARCHAR: {
        HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector)poi;
        HiveVarcharWritable hc = hcoi.getPrimitiveWritableObject(o);
        // use varchar's text field directly
        Text t = hc.getTextValue();
        serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
        return;
      }

      case BINARY: {
        BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
        BytesWritable ba = baoi.getPrimitiveWritableObject(o);
        byte[] toSer = new byte[ba.getLength()];
        System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
        serializeBytes(buffer, toSer, ba.getLength(), invert);
        return;
      }
      case  DATE: {
        DateObjectInspector doi = (DateObjectInspector) poi;
        int v = doi.getPrimitiveWritableObject(o).getDays();
        serializeInt(buffer, v, invert);
        return;
      }
      case TIMESTAMP: {
        TimestampObjectInspector toi = (TimestampObjectInspector) poi;
        TimestampWritable t = toi.getPrimitiveWritableObject(o);
        byte[] data = t.getBinarySortable();
        for (int i = 0; i < data.length; i++) {
          buffer.write(data[i], invert);
        }
        return;
      }
      case DECIMAL: {
        // decimals are encoded in three pieces:
        // sign: 1, 2 or 3 for smaller, equal or larger than 0 respectively
        // factor: Number that indicates the amount of digits you have to move
        // the decimal point left or right until the resulting number is smaller
        // than zero but has something other than 0 as the first digit.
        // digits: which is a string of all the digits in the decimal. If the number
        // is negative the binary string will be inverted to get the correct ordering.
        // Example: 0.00123
        // Sign is 3 (bigger than 0)
        // Factor is -2 (move decimal point 2 positions right)
        // Digits are: 123

        HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi;
        HiveDecimal dec = boi.getPrimitiveJavaObject(o);

        // get the sign of the big decimal
        int sign = dec.compareTo(HiveDecimal.ZERO);

    // we'll encode the absolute value (sign is separate)
    dec = dec.abs();

    // get the scale factor to turn big decimal into a decimal < 1
    int factor = dec.precision() - dec.scale();
    factor = sign == 1 ? factor : -factor;

        // convert the absolute big decimal to string
        dec.scaleByPowerOfTen(Math.abs(dec.scale()));
        String digits = dec.unscaledValue().toString();

        // finally write out the pieces (sign, scale, digits)
        buffer.write((byte) ( sign + 1), invert);
        buffer.write((byte) ((factor >> 24) ^ 0x80), invert);
        buffer.write((byte) ( factor >> 16), invert);
        buffer.write((byte) ( factor >> 8), invert);
        buffer.write((byte)   factor, invert);
        serializeBytes(buffer, digits.getBytes(decimalCharSet),
            digits.length(), sign == -1 ? !invert : invert);
        return;
      }

      default: {
        throw new RuntimeException("Unrecognized type: "
            + poi.getPrimitiveCategory());
      }
      }
    }
    case LIST: {
      ListObjectInspector loi = (ListObjectInspector) oi;
View Full Code Here


  private void serializeField(Object o, ObjectInspector oi, Object reuse)
      throws IOException {
    switch (oi.getCategory()) {
    case PRIMITIVE: {
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
      switch (poi.getPrimitiveCategory()) {
      case VOID: {
        return;
      }
      case BOOLEAN: {
        BooleanObjectInspector boi = (BooleanObjectInspector) poi;
        BooleanWritable r = reuse == null ? new BooleanWritable()
            : (BooleanWritable) reuse;
        r.set(boi.get(o));
        tbOut.write(r);
        return;
      }
      case BYTE: {
        ByteObjectInspector boi = (ByteObjectInspector) poi;
        ByteWritable r = reuse == null ? new ByteWritable()
            : (ByteWritable) reuse;
        r.set(boi.get(o));
        tbOut.write(r);
        return;
      }
      case SHORT: {
        ShortObjectInspector spoi = (ShortObjectInspector) poi;
        ShortWritable r = reuse == null ? new ShortWritable()
            : (ShortWritable) reuse;
        r.set(spoi.get(o));
        tbOut.write(r);
        return;
      }
      case INT: {
        IntObjectInspector ioi = (IntObjectInspector) poi;
        IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse;
        r.set(ioi.get(o));
        tbOut.write(r);
        return;
      }
      case LONG: {
        LongObjectInspector loi = (LongObjectInspector) poi;
        LongWritable r = reuse == null ? new LongWritable()
            : (LongWritable) reuse;
        r.set(loi.get(o));
        tbOut.write(r);
        return;
      }
      case FLOAT: {
        FloatObjectInspector foi = (FloatObjectInspector) poi;
        FloatWritable r = reuse == null ? new FloatWritable()
            : (FloatWritable) reuse;
        r.set(foi.get(o));
        tbOut.write(r);
        return;
      }
      case DOUBLE: {
        DoubleObjectInspector doi = (DoubleObjectInspector) poi;
        DoubleWritable r = reuse == null ? new DoubleWritable()
            : (DoubleWritable) reuse;
        r.set(doi.get(o));
        tbOut.write(r);
        return;
      }
      case STRING: {
        StringObjectInspector soi = (StringObjectInspector) poi;
        Text t = soi.getPrimitiveWritableObject(o);
        tbOut.write(t);
        return;
      }
      default: {
        throw new RuntimeException("Unrecognized type: "
            + poi.getPrimitiveCategory());
      }
      }
    }
    case LIST:
    case MAP:
View Full Code Here

          ObjectInspector foi = fields.get(k).getFieldObjectInspector();
          ColumnVector currentColVector = batch.cols[k];

          switch (foi.getCategory()) {
          case PRIMITIVE: {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi;
            if (!currentColVector.noNulls
                && (currentColVector.isRepeating || currentColVector.isNull[rowIndex])) {
              // The column is null hence write null value
              serializeVectorStream.write(new byte[0], 0, 0);
            } else {
              // If here then the vector value is not null.
              if (currentColVector.isRepeating) {
                // If the vector has repeating values then set rowindex to zero
                rowIndex = 0;
              }

              switch (poi.getPrimitiveCategory()) {
              case BOOLEAN: {
                LongColumnVector lcv = (LongColumnVector) batch.cols[k];
                // In vectorization true is stored as 1 and false as 0
                boolean b = lcv.vector[rowIndex] == 1 ? true : false;
                if (b) {
                  serializeVectorStream.write(LazyUtils.trueBytes, 0, LazyUtils.trueBytes.length);
                } else {
                  serializeVectorStream.write(LazyUtils.trueBytes, 0, LazyUtils.trueBytes.length);
                }
              }
                break;
              case BYTE:
              case SHORT:
              case INT:
              case LONG:
                LongColumnVector lcv = (LongColumnVector) batch.cols[k];
                LazyLong.writeUTF8(serializeVectorStream, lcv.vector[rowIndex]);
                break;
              case FLOAT:
              case DOUBLE:
                DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[k];
                ByteBuffer b = Text.encode(String.valueOf(dcv.vector[rowIndex]));
                serializeVectorStream.write(b.array(), 0, b.limit());
                break;
              case STRING:
                BytesColumnVector bcv = (BytesColumnVector) batch.cols[k];
                LazyUtils.writeEscaped(serializeVectorStream, bcv.vector[rowIndex],
                    bcv.start[rowIndex],
                    bcv.length[rowIndex],
                    serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams
                        .getNeedsEscape());
                break;
              case TIMESTAMP:
                LongColumnVector tcv = (LongColumnVector) batch.cols[k];
                long timeInNanoSec = tcv.vector[rowIndex];
                Timestamp t = new Timestamp(0);
                TimestampUtils.assignTimeInNanoSec(timeInNanoSec, t);
                TimestampWritable tw = new TimestampWritable();
                tw.set(t);
                LazyTimestamp.writeUTF8(serializeVectorStream, tw);
                break;
              case DATE:
                LongColumnVector dacv = (LongColumnVector) batch.cols[k];
                DateWritable daw = new DateWritable((int) dacv.vector[rowIndex]);
                LazyDate.writeUTF8(serializeVectorStream, daw);
                break;
              default:
                throw new UnsupportedOperationException(
                    "Vectorizaton is not supported for datatype:"
                        + poi.getPrimitiveCategory());
              }
            }
            break;
          }
          case LIST:
View Full Code Here

    return vca;
  }

  public static VectorColumnAssign buildObjectAssign(VectorizedRowBatch outputBatch,
      int outColIndex, ObjectInspector objInspector) throws HiveException {
    PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector;
    VectorColumnAssign outVCA = null;
    ColumnVector destCol = outputBatch.cols[outColIndex];
    if (destCol instanceof LongColumnVector) {
      switch(poi.getPrimitiveCategory()) {
      case BOOLEAN:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              BooleanWritable bw = (BooleanWritable) val;
              assignLong(bw.get() ? 1:0, destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case BYTE:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              ByteWritable bw = (ByteWritable) val;
              assignLong(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case SHORT:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              ShortWritable bw = (ShortWritable) val;
              assignLong(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case INT:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              IntWritable bw = (IntWritable) val;
              assignLong(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case LONG:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              LongWritable bw = (LongWritable) val;
              assignLong(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case TIMESTAMP:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              TimestampWritable bw = (TimestampWritable) val;
              Timestamp t = bw.getTimestamp();
              assignLong(TimestampUtils.getTimeNanoSec(t), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case DATE:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              DateWritable bw = (DateWritable) val;
              assignLong(bw.getDays(), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      default:
        throw new HiveException("Incompatible Long vector column and primitive category " +
            poi.getPrimitiveCategory());
      }
    }
    else if (destCol instanceof DoubleColumnVector) {
      switch(poi.getPrimitiveCategory()) {
      case DOUBLE:
        outVCA = new VectorDoubleColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              DoubleWritable bw = (DoubleWritable) val;
              assignDouble(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (DoubleColumnVector) destCol);
        break;
      case FLOAT:
        outVCA = new VectorDoubleColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              FloatWritable bw = (FloatWritable) val;
              assignDouble(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (DoubleColumnVector) destCol);
        break;
      default:
        throw new HiveException("Incompatible Double vector column and primitive category " +
            poi.getPrimitiveCategory());
      }
    }
    else if (destCol instanceof BytesColumnVector) {
      switch(poi.getPrimitiveCategory()) {
      case STRING:
        outVCA = new VectorBytesColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              Text bw = (Text) val;
              byte[] bytes = bw.getBytes();
              assignBytes(bytes, 0, bytes.length, destIndex);
            }
          }
        }.init(outputBatch, (BytesColumnVector) destCol);
        break;
      default:
        throw new HiveException("Incompatible Bytes vector column and primitive category " +
            poi.getPrimitiveCategory());
      }
    }
    else if (destCol instanceof DecimalColumnVector) {
      switch(poi.getPrimitiveCategory()) {
      case DECIMAL:
        outVCA = new VectorDecimalColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
              if (val == null) {
                assignNull(destIndex);
              }
              else {
                HiveDecimalWritable hdw = (HiveDecimalWritable) val;
                assignDecimal(hdw, destIndex);
              }
            }
          }.init(outputBatch, (DecimalColumnVector) destCol);
          break;
        default:
          throw new HiveException("Incompatible Decimal vector column and primitive category " +
              poi.getPrimitiveCategory());
        }
    }
    else {
      throw new HiveException("Unknown vector column type " + destCol.getClass().getName());
    }
View Full Code Here

    };
    DeferredObject[] args = {
        new DeferredJavaObject(input)
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.longTypeInfo, oi.getTypeInfo());
    LongWritable res = (LongWritable) udf.evaluate(args);
    Assert.assertEquals(747L, res.get());
  }
View Full Code Here

    };
    DeferredObject[] args = {
        new DeferredJavaObject(input)
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.longTypeInfo, oi.getTypeInfo());
    LongWritable res = (LongWritable) udf.evaluate(args);
    Assert.assertEquals(3234747L, res.get());
  }
View Full Code Here

    };
    DeferredObject[] args = {
        new DeferredJavaObject(input)
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.longTypeInfo, oi.getTypeInfo());
    LongWritable res = (LongWritable) udf.evaluate(args);
    Assert.assertEquals(324L, res.get());
  }
View Full Code Here

    };
    DeferredObject[] args = {
        new DeferredJavaObject(input)
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.longTypeInfo, oi.getTypeInfo());
    LongWritable res = (LongWritable) udf.evaluate(args);
    Assert.assertEquals(32301L, res.get());
  }
View Full Code Here

    };
    DeferredObject[] args = {
        new DeferredJavaObject(input)
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(6, 0), oi.getTypeInfo());
    HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args);
    Assert.assertEquals(HiveDecimal.create("32301"), res.getHiveDecimal());
  }
View Full Code Here

    };
    DeferredObject[] args = {
        new DeferredJavaObject(input)
    };

    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.longTypeInfo, oi.getTypeInfo());
    LongWritable res = (LongWritable) udf.evaluate(args);
    Assert.assertEquals(32301L, res.get());
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.