Examples of org.apache.phoenix.util.TrustedByteArrayOutputStream

org.apache.phoenix.util.TrustedByteArrayOutputStream
Derived version of {@link java.io.ByteArrayOutputStream} that provides accessto underlying byte array buffer so that it doesn't have to be copied @since 0.1

        while (indexes.hasNext()) {
            nIndexes++;
            PTable index = indexes.next();
            estimatedSize += index.getIndexMaintainer(dataTable).getEstimatedByteSize();
        }
        TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedSize + 1);
        DataOutput output = new DataOutputStream(stream);
        try {
            // Encode data table salting in sign of number of indexes
            WritableUtils.writeVInt(output, nIndexes * (dataTable.getBucketNum() == null ? 1 : -1));
            // Write out data row key schema once, since it's the same for all index maintainers
            dataTable.getRowKeySchema().write(output);
            indexes = nonDisabledIndexIterator(dataTable.getIndexes().iterator());
            while (indexes.hasNext()) {
                    indexes.next().getIndexMaintainer(dataTable).write(output);
            }
        } catch (IOException e) {
            throw new RuntimeException(e); // Impossible
        }
        ptr.set(stream.getBuffer(), 0, stream.size());
    }

View Full Code Here

        this.indexWALDisabled = indexWALDisabled;
    }


    public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr)  {
        ImmutableBytesWritable ptr = new ImmutableBytesWritable();
        TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
        DataOutput output = new DataOutputStream(stream);
        try {
            if (nIndexSaltBuckets > 0) {
                output.write(0); // will be set at end to index salt byte
            }
            // The dataRowKeySchema includes the salt byte field,
            // so we must adjust for that here.
            int dataPosOffset = isDataTableSalted ? 1 : 0 ;
            int nIndexedColumns = getIndexPkColumnCount();
            int[][] dataRowKeyLocator = new int[2][nIndexedColumns];
            // Skip data table salt byte
            int maxRowKeyOffset = rowKeyPtr.getOffset() + rowKeyPtr.getLength();
            dataRowKeySchema.iterator(rowKeyPtr, ptr, dataPosOffset);
            if (isMultiTenant) {
                dataRowKeySchema.next(ptr, dataPosOffset, maxRowKeyOffset);
                output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
                    output.writeByte(QueryConstants.SEPARATOR_BYTE);
                }
                dataPosOffset++;
            }
            if (viewIndexId != null) {
                output.write(viewIndexId);
            }
            
            BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
            // Write index row key
            for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
                Boolean hasValue=dataRowKeySchema.next(ptr, i, maxRowKeyOffset);
                // Ignore view constants from the data table, as these
                // don't need to appear in the index (as they're the
                // same for all rows in this index)
                if (!viewConstantColumnBitSet.get(i)) {
                    int pos = rowKeyMetaData.getIndexPkPosition(i-dataPosOffset);
                    if (Boolean.TRUE.equals(hasValue)) {
                        dataRowKeyLocator[0][pos] = ptr.getOffset();
                        dataRowKeyLocator[1][pos] = ptr.getLength();
                    } else {
                        dataRowKeyLocator[0][pos] = 0;
                        dataRowKeyLocator[1][pos] = 0;
                    }
                }
            }
            BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
            int j = 0;
            Iterator<ColumnReference> iterator = indexedColumns.iterator();
            for (int i = 0; i < nIndexedColumns; i++) {
                PDataType dataColumnType;
                boolean isNullable = true;
                boolean isDataColumnInverted = false;
                SortOrder dataSortOrder = SortOrder.getDefault();
                if (dataPkPosition[i] == -1) {
                    dataColumnType = indexedColumnTypes.get(j);
                    ImmutableBytesPtr value = valueGetter.getLatestValue(iterator.next());
                    if (value == null) {
                        ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
                    } else {
                        ptr.set(value.copyBytesIfNecessary());
                    }
                    j++;
               } else {
                    Field field = dataRowKeySchema.getField(dataPkPosition[i]);
                    dataColumnType = field.getDataType();
                    ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]);
                    dataSortOrder = field.getSortOrder();
                    isDataColumnInverted = dataSortOrder != SortOrder.ASC;
                    isNullable = field.isNullable();
                }
                PDataType indexColumnType = IndexUtil.getIndexColumnDataType(isNullable, dataColumnType);
                boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType) ;
                if (isBytesComparable && isDataColumnInverted == descIndexColumnBitSet.get(i)) {
                    output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                } else {
                    if (!isBytesComparable)  {
                        indexColumnType.coerceBytes(ptr, dataColumnType, dataSortOrder, SortOrder.getDefault());
                    }
                    if (descIndexColumnBitSet.get(i) != isDataColumnInverted) {
                        writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
                    } else {
                        output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                    }
                }
                if (!indexColumnType.isFixedWidth()) {
                    output.writeByte(QueryConstants.SEPARATOR_BYTE);
                }
            }
            int length = stream.size();
            int minLength = length - maxTrailingNulls;
            byte[] indexRowKey = stream.getBuffer();
            // Remove trailing nulls
            while (length > minLength && indexRowKey[length-1] == QueryConstants.SEPARATOR_BYTE) {
                length--;
            }
            if (nIndexSaltBuckets > 0) {
                // Set salt byte
                byte saltByte = SaltingUtil.getSaltingByte(indexRowKey, SaltingUtil.NUM_SALTING_BYTES, length-SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
                indexRowKey[0] = saltByte;
            }
            return indexRowKey.length == length ? indexRowKey : Arrays.copyOf(indexRowKey, length);
        } catch (IOException e) {
            throw new RuntimeException(e); // Impossible
        } finally {
            try {
                stream.close();
            } catch (IOException e) {
                throw new RuntimeException(e); // Impossible
            }
        }
    }

View Full Code Here

        estimatedSize = Math.min(estimatedSize, maxSize);
        if (estimatedSize > Integer.MAX_VALUE) {
            throw new IllegalStateException("Estimated size(" + estimatedSize + ") must not be greater than Integer.MAX_VALUE(" + Integer.MAX_VALUE + ")");
        }
        try {
            TrustedByteArrayOutputStream baOut = new TrustedByteArrayOutputStream((int)estimatedSize);
            DataOutputStream out = new DataOutputStream(baOut);
            // Write onExpressions first, for hash key evaluation along with deserialization
            out.writeInt(onExpressions.size());
            for (Expression expression : onExpressions) {
                WritableUtils.writeVInt(out, ExpressionType.valueOf(expression).ordinal());
                expression.write(out);                
            }
            int exprSize = baOut.size() + Bytes.SIZEOF_INT;
            out.writeInt(exprSize);
            int nRows = 0;
            out.writeInt(nRows); // In the end will be replaced with total number of rows            
            for (Tuple result = iterator.next(); result != null; result = iterator.next()) {
                if (projector != null) {
                    result = projector.projectResults(result);
                }
                TupleUtil.write(result, out);
                if (baOut.size() > maxSize) {
                    throw new MaxServerCacheSizeExceededException("Size of hash cache (" + baOut.size() + " bytes) exceeds the maximum allowed size (" + maxSize + " bytes)");
                }
                nRows++;
            }
            TrustedByteArrayOutputStream sizeOut = new TrustedByteArrayOutputStream(Bytes.SIZEOF_INT);
            DataOutputStream dataOut = new DataOutputStream(sizeOut);
            try {
                dataOut.writeInt(nRows);
                dataOut.flush();
                byte[] cache = baOut.getBuffer();
                // Replace number of rows written above with the correct value.
                System.arraycopy(sizeOut.getBuffer(), 0, cache, exprSize, sizeOut.size());
                // Reallocate to actual size plus compressed buffer size (which is allocated below)
                int maxCompressedSize = Snappy.maxCompressedLength(baOut.size());
                byte[] compressed = new byte[maxCompressedSize]; // size for worst case
                int compressedSize = Snappy.compress(baOut.getBuffer(), 0, baOut.size(), compressed, 0);
                // Last realloc to size of compressed buffer.

View Full Code Here

        int nValues = values.length;
        while (nValues > 0 && (values[nValues-1] == null || values[nValues-1].length == 0)) {
            nValues--;
        }
        int i = 0;
        TrustedByteArrayOutputStream os = new TrustedByteArrayOutputStream(SchemaUtil.estimateKeyLength(this));
        try {
            Integer bucketNum = this.getBucketNum();
            if (bucketNum != null) {
                // Write place holder for salt byte
                i++;
                os.write(QueryConstants.SEPARATOR_BYTE_ARRAY);
            }
            List<PColumn> columns = getPKColumns();
            int nColumns = columns.size();
            PDataType type = null;
            while (i < nValues && i < nColumns) {
                // Separate variable length column values in key with zero byte
                if (type != null && !type.isFixedWidth()) {
                    os.write(SEPARATOR_BYTE);
                }
                PColumn column = columns.get(i);
                type = column.getDataType();
                // This will throw if the value is null and the type doesn't allow null
                byte[] byteValue = values[i++];
                if (byteValue == null) {
                    byteValue = ByteUtil.EMPTY_BYTE_ARRAY;
                }
                // An empty byte array return value means null. Do this,
                // since a type may have muliple representations of null.
                // For example, VARCHAR treats both null and an empty string
                // as null. This way we don't need to leak that part of the
                // implementation outside of PDataType by checking the value
                // here.
                if (byteValue.length == 0 && !column.isNullable()) { 
                    throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
                }
                Integer  maxLength = column.getMaxLength();
                if (maxLength != null && type.isFixedWidth() && byteValue.length <= maxLength) {
                    byteValue = StringUtil.padChar(byteValue, maxLength);
                } else if (maxLength != null && byteValue.length > maxLength) {
                    throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not exceed " + maxLength + " bytes (" + SchemaUtil.toString(type, byteValue) + ")");
                }
                os.write(byteValue, 0, byteValue.length);
            }
            // If some non null pk values aren't set, then throw
            if (i < nColumns) {
                PColumn column = columns.get(i);
                type = column.getDataType();
                if (type.isFixedWidth() || !column.isNullable()) {
                    throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
                }
            }
            if (nValues == 0) { 
                throw new ConstraintViolationException("Primary key may not be null ("+ name.getString() + ")");
            }
            byte[] buf = os.getBuffer();
            int size = os.size();
            if (bucketNum != null) {
                buf[0] = SaltingUtil.getSaltingByte(buf, 1, size-1, bucketNum);
            }
            key.set(buf,0,size);
            return i;
        } finally {
            try {
                os.close();
            } catch (IOException e) {
                throw new RuntimeException(e); // Impossible
            }
        }
    }

View Full Code Here

        super(children);
        init(baseType);
        estimatedSize = PArrayDataType.estimateSize(this.children.size(), this.baseType);
        if (!this.baseType.isFixedWidth()) {
            offsetPos = new int[children.size()];
            byteStream = new TrustedByteArrayOutputStream(estimatedSize);
        } else {
            byteStream = new TrustedByteArrayOutputStream(estimatedSize);
        }
            
    }

View Full Code Here

                }
                if (expressionCount == 1) {
                    ptr.set(ptrs[0].get(), ptrs[0].getOffset(), ptrs[0].getLength());
                    return true;
                }
                TrustedByteArrayOutputStream output = new TrustedByteArrayOutputStream(estimatedByteSize);
                try {
                    boolean previousCarryOver = false;
                    for (int i = 0; i< expressionCount; i++) {
                        Expression child = getChildren().get(i);
                        PDataType childType = child.getDataType();
                        ImmutableBytesWritable tempPtr = ptrs[i];
                        if (tempPtr == null) {
                            // Since we have a null and have no representation for null,
                            // we must decrement the value of the current. Otherwise,
                            // we'd have an ambiguity if this value happened to be the
                            // min possible value.
                            previousCarryOver = childType == null || childType.isFixedWidth();
                            int bytesToWrite = getExpressionByteCount(child);
                            for (int m = 0; m < bytesToWrite; m++) {
                                output.write(QueryConstants.SEPARATOR_BYTE);
                            }
                        } else {
                            output.write(tempPtr.get(), tempPtr.getOffset(), tempPtr.getLength());
                            if (!childType.isFixedWidth()) {
                                output.write(QueryConstants.SEPARATOR_BYTE);
                            }
                            if (previousCarryOver) {
                                previousCarryOver = !ByteUtil.previousKey(output.getBuffer(), output.size());
                            }
                        }
                    }
                    int outputSize = output.size();
                    byte[] outputBytes = output.getBuffer();
                    for (int k = expressionCount -1 ; 
                            k >=0 &&  getChildren().get(k).getDataType() != null && !getChildren().get(k).getDataType().isFixedWidth() && outputBytes[outputSize-1] == QueryConstants.SEPARATOR_BYTE ; k--) {
                        outputSize--;
                    }
                    ptr.set(outputBytes, 0, outputSize);
                    return true;
                } finally {
                    output.close();
                }
            }  
            return false;
        } catch (IOException e) {
            throw new RuntimeException(e); //Impossible.

View Full Code Here

    PhoenixArray arr = ((PhoenixArray)object);
        int noOfElements = arr.numElements;
        if(noOfElements == 0) {
          return ByteUtil.EMPTY_BYTE_ARRAY;
        }
        TrustedByteArrayOutputStream byteStream = null;
    if (!baseType.isFixedWidth()) {
          Pair<Integer, Integer> nullsVsNullRepeationCounter = new Pair<Integer, Integer>();
          int size = estimateByteSize(object, nullsVsNullRepeationCounter,
                  PDataType.fromTypeId((baseType.getSqlType() + PDataType.ARRAY_TYPE_BASE)));
        size += ((2 * Bytes.SIZEOF_BYTE) + (noOfElements - nullsVsNullRepeationCounter.getFirst()) * Bytes.SIZEOF_BYTE)
                                    + (nullsVsNullRepeationCounter.getSecond() * 2 * Bytes.SIZEOF_BYTE);
        // Assume an offset array that fit into Short.MAX_VALUE.  Also not considering nulls that could be > 255
        // In both of these cases, finally an array copy would happen
        int capacity = noOfElements * Bytes.SIZEOF_SHORT;
        // Here the int for noofelements, byte for the version, int for the offsetarray position and 2 bytes for the end seperator
            byteStream = new TrustedByteArrayOutputStream(size + capacity + Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE +  Bytes.SIZEOF_INT);
    } else {
        int size = arr.getMaxLength() * noOfElements;
        // Here the int for noofelements, byte for the version
        byteStream = new TrustedByteArrayOutputStream(size);
    }
    DataOutputStream oStream = new DataOutputStream(byteStream);
    // Handles bit inversion also
    return createArrayBytes(byteStream, oStream, (PhoenixArray)object, noOfElements, baseType, sortOrder);
  }

View Full Code Here

        int nValues = values.length;
        while (nValues > 0 && (values[nValues-1] == null || values[nValues-1].length == 0)) {
            nValues--;
        }
        int i = 0;
        TrustedByteArrayOutputStream os = new TrustedByteArrayOutputStream(SchemaUtil.estimateKeyLength(this));
        try {
            Integer bucketNum = this.getBucketNum();
            if (bucketNum != null) {
                // Write place holder for salt byte
                i++;
                os.write(QueryConstants.SEPARATOR_BYTE_ARRAY);
            }
            List<PColumn> columns = getPKColumns();
            int nColumns = columns.size();
            PDataType type = null;
            while (i < nValues && i < nColumns) {
                // Separate variable length column values in key with zero byte
                if (type != null && !type.isFixedWidth()) {
                    os.write(SEPARATOR_BYTE);
                }
                PColumn column = columns.get(i);
                type = column.getDataType();
                // This will throw if the value is null and the type doesn't allow null
                byte[] byteValue = values[i++];
                if (byteValue == null) {
                    byteValue = ByteUtil.EMPTY_BYTE_ARRAY;
                }
                // An empty byte array return value means null. Do this,
                // since a type may have muliple representations of null.
                // For example, VARCHAR treats both null and an empty string
                // as null. This way we don't need to leak that part of the
                // implementation outside of PDataType by checking the value
                // here.
                if (byteValue.length == 0 && !column.isNullable()) { 
                    throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
                }
                Integer byteSize = column.getByteSize();
                if (type.isFixedWidth() && byteValue.length <= byteSize) {
                    byteValue = StringUtil.padChar(byteValue, byteSize);
                } else if (byteSize != null && byteValue.length > byteSize) {
                    throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not exceed " + byteSize + " bytes (" + SchemaUtil.toString(type, byteValue) + ")");
                }
                os.write(byteValue, 0, byteValue.length);
            }
            // If some non null pk values aren't set, then throw
            if (i < nColumns) {
                PColumn column = columns.get(i);
                type = column.getDataType();
                if (type.isFixedWidth() || !column.isNullable()) {
                    throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
                }
            }
            byte[] buf = os.getBuffer();
            int size = os.size();
            if (bucketNum != null) {
                buf[0] = SaltingUtil.getSaltingByte(buf, 1, size-1, bucketNum);
            }
            key.set(buf,0,size);
            return i;
        } finally {
            try {
                os.close();
            } catch (IOException e) {
                throw new RuntimeException(e); // Impossible
            }
        }
    }

View Full Code Here

                }
                if (expressionCount == 1) {
                    ptr.set(ptrs[0].get(), ptrs[0].getOffset(), ptrs[0].getLength());
                    return true;
                }
                TrustedByteArrayOutputStream output = new TrustedByteArrayOutputStream(estimatedByteSize);
                try {
                    boolean previousCarryOver = false;
                    for (int i = 0; i< expressionCount; i++) {
                        Expression child = getChildren().get(i);
                        PDataType childType = child.getDataType();
                        ImmutableBytesWritable tempPtr = ptrs[i];
                        if (tempPtr == null) {
                            // Since we have a null and have no representation for null,
                            // we must decrement the value of the current. Otherwise,
                            // we'd have an ambiguity if this value happened to be the
                            // min possible value.
                            previousCarryOver = childType == null || childType.isFixedWidth();
                            int bytesToWrite = getExpressionByteCount(child);
                            for (int m = 0; m < bytesToWrite; m++) {
                                output.write(QueryConstants.SEPARATOR_BYTE);
                            }
                        } else {
                            output.write(tempPtr.get(), tempPtr.getOffset(), tempPtr.getLength());
                            if (!childType.isFixedWidth()) {
                                output.write(QueryConstants.SEPARATOR_BYTE);
                            }
                            if (previousCarryOver) {
                                previousCarryOver = !ByteUtil.previousKey(output.getBuffer(), output.size());
                            }
                        }
                    }
                    int outputSize = output.size();
                    byte[] outputBytes = output.getBuffer();
                    for (int k = expressionCount -1 ; 
                            k >=0 &&  getChildren().get(k).getDataType() != null && !getChildren().get(k).getDataType().isFixedWidth() && outputBytes[outputSize-1] == QueryConstants.SEPARATOR_BYTE ; k--) {
                        outputSize--;
                    }
                    ptr.set(outputBytes, 0, outputSize);
                    return true;
                } finally {
                    output.close();
                }
            }  
            return false;
        } catch (IOException e) {
            throw new RuntimeException(e); //Impossible.

View Full Code Here

        while (indexes.hasNext()) {
            nIndexes++;
            PTable index = indexes.next();
            estimatedSize += index.getIndexMaintainer(dataTable).getEstimatedByteSize();
        }
        TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedSize + 1);
        DataOutput output = new DataOutputStream(stream);
        try {
            // Encode data table salting in sign of number of indexes
            WritableUtils.writeVInt(output, nIndexes * (dataTable.getBucketNum() == null ? 1 : -1));
            // Write out data row key schema once, since it's the same for all index maintainers
            dataTable.getRowKeySchema().write(output);
            indexes = nonDisabledIndexIterator(dataTable.getIndexes().iterator());
            while (indexes.hasNext()) {
                    indexes.next().getIndexMaintainer(dataTable).write(output);
            }
        } catch (IOException e) {
            throw new RuntimeException(e); // Impossible
        }
        ptr.set(stream.getBuffer(), 0, stream.size());
    }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.phoenix.util.TrustedByteArrayOutputStream

org.apache.phoenix.expression.ArrayConstructorExpression

org.apache.phoenix.expression.RowValueConstructorExpression

org.apache.phoenix.index.IndexMaintainer

org.apache.phoenix.join.HashCacheClient

org.apache.phoenix.schema.PArrayDataType

org.apache.phoenix.schema.PTableImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.