Package com.addthis.bundle.io

Source Code of com.addthis.bundle.io.DataChannelCodec$ObjectIndexMap

/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.addthis.bundle.io;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import java.util.HashMap;

import com.addthis.basis.util.Bytes;

import com.addthis.bundle.core.Bundle;
import com.addthis.bundle.core.BundleField;
import com.addthis.bundle.table.DataTable;
import com.addthis.bundle.value.ValueArray;
import com.addthis.bundle.value.ValueCustom;
import com.addthis.bundle.value.ValueFactory;
import com.addthis.bundle.value.ValueMap;
import com.addthis.bundle.value.ValueMapEntry;
import com.addthis.bundle.value.ValueObject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* A utility for creating/consuming binary encodings of DataTable to/from
* streams.
* <p/>
* TODO need a mid-stream dictionary reset detector/switch
*/
public final class DataChannelCodec {

    private static final Logger log = LoggerFactory.getLogger(DataChannelCodec.class);

    private static final HashMap<Integer, TYPE> typeMap = new HashMap<>();
    private static final TYPE bundleTypeInit = TYPE.BUNDLE_INIT;
    private static final TYPE bundleTypeStart = TYPE.BUNDLE_START;
    private static final FieldIndexMap FIMNull = new FIMNull();
    private static final ClassIndexMap CIMNull = new CIMNull();

    private static enum TYPE {
        NULL(0), STRING(1), BYTES(2), LONG(3), LONG_NEG(4), LONG_BIG(5),
        DOUBLE(6), ARRAY(7), MAP(8), CUSTOM_INDEX(9), CUSTOM_CLASS(10),
        BUNDLE_INIT(11), BUNDLE_START(12), BUNDLE_FIELD_INDEX(13), BUNDLE_FIELD_NAME(14), BUNDLE_END(15);

        private final int val;

        TYPE(final int val) {
            this.val = val;
            typeMap.put(val, this);
        }

        @Override
        public String toString() {
            return val + "=" + name();
        }
    }

    /** */
    public static interface ObjectIndexMap<T> {

        /**
         * @return null if no index exists for this class
         */
        public Integer getObjectIndex(T type);

        /**
         * @return index or null if index already exists
         */
        public Integer createObjectIndex(T type);

        /**
         * @return index or null if index already exists
         */
        public Integer setObjectIndex(int index, T type);

        /**
         * @return class registered to this index or null if none
         */
        public T getObject(Integer index);

        /**
         * @return number of entries in map
         */
        public int size();

        /**
         * reset map state
         */
        public void reset();
    }

    public static interface FieldIndexMap extends ObjectIndexMap<BundleField> {

    }

    public static interface ClassIndexMap extends ObjectIndexMap<Class<? extends ValueObject>> {

    }

    /**
     * for non-persisted, transient sends/receives
     */
    public static ClassIndexMap createClassIndexMap() {
        return new CIM();
    }

    /**
     * for non-persisted, transient sends/receives
     */
    public static FieldIndexMap createFieldIndexMap() {
        return new FIM();
    }

    /**
     * for stateless encoding
     */
    public static byte[] encodeBundle(Bundle row) throws IOException {
        return encodeBundle(row, FIMNull, CIMNull);
    }

    /**
     * used in QueryChannelServer
     */
    public static byte[] encodeBundle(Bundle row, FieldIndexMap fieldMap, ClassIndexMap classIndex) throws IOException {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        encodeBundle(row, out, fieldMap, classIndex);
        return out.toByteArray();
    }

    /**
     * used in ResultDiskBacked
     */
    public static void encodeBundle(Bundle row, OutputStream out, FieldIndexMap fieldMap, ClassIndexMap classMap) throws IOException {
        if (fieldMap.size() == 0 && classMap.size() == 0) {
            out.write(TYPE.BUNDLE_INIT.val);
        } else {
            out.write(TYPE.BUNDLE_START.val);
        }
        for (BundleField field : row) {
            Integer fieldIndex = fieldMap.getObjectIndex(field);
            if (fieldIndex == null) {
                fieldIndex = fieldMap.createObjectIndex(field);
                out.write(TYPE.BUNDLE_FIELD_NAME.val);
                Bytes.writeLength(fieldIndex, out);
                Bytes.writeString(field.getName(), out);
            } else {
                out.write(TYPE.BUNDLE_FIELD_INDEX.val);
                Bytes.writeLength(fieldIndex, out);
            }
            ValueObject val = row.getValue(field);
            encodeValue(val, out, classMap);
        }
        out.write(TYPE.BUNDLE_END.val);
    }

    /**
     * encode a single value to a stream
     */
    public static void encodeValue(ValueObject val, OutputStream out, ClassIndexMap classIndex) throws IOException {
        if (val == null) {
            out.write(TYPE.NULL.val);
            return;
        }
        ValueObject.TYPE objectType = val.getObjectType();
        switch (objectType) {
            case CUSTOM:
                ValueCustom custom = val.asCustom();
                Class<? extends ValueObject> type = custom.getContainerClass();
                Integer classID = classIndex.getObjectIndex(type);
                if (classID == null) {
                    classID = classIndex.createObjectIndex(type);
                    out.write(TYPE.CUSTOM_CLASS.val);
                    Bytes.writeLength(classID, out);
                    Bytes.writeString(type.getName(), out);
                } else {
                    out.write(TYPE.CUSTOM_INDEX.val);
                    Bytes.writeLength(classID, out);
                }
                encodeValue(custom.asMap(), out, classIndex);
                break;
            case MAP:
                ValueMap map = val.asMap();
                out.write(TYPE.MAP.val);
                Bytes.writeLength(map.size(), out);
                for (ValueMapEntry e : map) {
                    encodeValue(ValueFactory.create(e.getKey()), out, classIndex);
                    encodeValue(e.getValue(), out, classIndex);
                }
                break;
            case ARRAY:
                out.write(TYPE.ARRAY.val);
                ValueArray arr = val.asArray();
                Bytes.writeLength(arr.size(), out);
                for (ValueObject vo : arr) {
                    encodeValue(vo, out, classIndex);
                }
                break;
            case STRING:
                out.write(TYPE.STRING.val);
                Bytes.writeString(val.asString().getString(), out);
                break;
            case BYTES:
                out.write(TYPE.BYTES.val);
                Bytes.writeBytes(val.asBytes().getBytes(), out);
                break;
            case INT:
                long lv = val.asLong().getLong();
                // over 2^48, direct bytes are more efficient
                if (lv > 281474976710656L) {
                    out.write(TYPE.LONG_BIG.val);
                    Bytes.writeLong(lv, out);
                    break;
                }
                if (lv >= 0) {
                    out.write(TYPE.LONG.val);
                } else {
                    out.write(TYPE.LONG_NEG.val);
                    lv = -lv;
                }
                Bytes.writeLength(lv, out);
                break;
            case FLOAT:
                long dv = Double.doubleToLongBits(val.asDouble().getDouble());
                out.write(TYPE.DOUBLE.val);
                Bytes.writeLong(dv, out);
                break;
            default:
                log.error("Unknown object type " + objectType);
                throw new IOException("Unknown object type " + objectType);
        }
    }

    /**
     * for stateless decoding.  bytes MUST come from a stateless encode.
     */
    public static Bundle decodeBundle(Bundle bundle, byte row[]) throws IOException {
        return decodeBundle(bundle, new ByteArrayInputStream(row), FIMNull, CIMNull);
    }

    /**
     * used in QueryChannelServer
     */
    public static Bundle decodeBundle(Bundle bundle, byte row[], FieldIndexMap fieldMap, ClassIndexMap classMap) throws IOException {
        return decodeBundle(bundle, new ByteArrayInputStream(row), fieldMap, classMap);
    }

    /**
     * used in ResultDiskBacked
     */
    public static Bundle decodeBundle(Bundle bundle, InputStream in, FieldIndexMap fieldMap, ClassIndexMap classMap) throws IOException {
        int t = in.read();
        if (t < 0) {
            return null;
        }
        TYPE type = typeMap.get(t);
        if (type == bundleTypeInit) {
            // first bundle in a stream or mid-stream for stream appends
            fieldMap.reset();
            classMap.reset();
        } else if (type == bundleTypeStart) {
            // normal for streams after first bundle
        } else {
            throw new IOException("type mismatch (" + t + " = " + type + ") != valid Bundle type");
        }
        int typever = 0;
        try {
            loop:
            while (true) {
                typever = in.read();
                switch (type = typeMap.get(typever)) {
                    case BUNDLE_FIELD_INDEX:
                        int fi = 0;
                        ValueObject vo = null;
                        BundleField field;
                        try {
                            fi = in.read();
                            vo = decodeValue(in, classMap);
                            field = fieldMap.getObject(fi);
                            bundle.setValue(field, vo);
                        } catch (RuntimeException ex) {
                            log.error("field index failure @ " + fi + " val=" + vo);
                            throw ex;
                        }
                        break;
                    case BUNDLE_FIELD_NAME:
                        int index = (int) Bytes.readLength(in);
                        field = bundle.getFormat().getField(Bytes.readString(in));
                        fieldMap.setObjectIndex(index, field);
                        bundle.setValue(field, decodeValue(in, classMap));
                        break;
                    case BUNDLE_END:
                        break loop;
                    default:
                        throw new IOException("type mismatch " + type + " not valid bundle field type");
                }
            }
        } catch (RuntimeException ex) {
            String msg = "decode error from " + in + "\n";
            msg += "  typever = " + typever + "\n";
            msg += "  type = " + type + "\n";
            msg += "  bundle = " + bundle + "\n";
            msg += "  fields = " + fieldMap + "\n";
            msg += "  classz = " + classMap + "\n";
            log.error(msg);
            throw ex;
        }
        return bundle;
    }

    public static ValueObject decodeValue(InputStream in, ClassIndexMap classMap) throws IOException {
        int typeInteger = in.read();
        TYPE type = typeMap.get(typeInteger);
        if (type == null) {
            throw new IOException("Null object type, integer code is " + typeInteger);
        }
        switch (type) {
            case NULL:
                return null;
            case BYTES:
                return ValueFactory.create(Bytes.readBytes(in));
            case LONG:
                return ValueFactory.create(Bytes.readLength(in));
            case LONG_NEG:
                return ValueFactory.create(-Bytes.readLength(in));
            case LONG_BIG:
                return ValueFactory.create(Bytes.readLong(in));
            case DOUBLE:
                return ValueFactory.create(Double.longBitsToDouble(Bytes.readLong(in)));
            case STRING:
                return ValueFactory.create(Bytes.readString(in));
            case ARRAY:
                int len = (int) Bytes.readLength(in);
                ValueArray arr = ValueFactory.createArray(len);
                for (int i = 0; i < len; i++) {
                    arr.append(decodeValue(in, classMap));
                }
                return arr;
            case MAP:
                ValueMap map = ValueFactory.createMap();
                long count = Bytes.readLength(in);
                while (count-- > 0) {
                    ValueObject key = decodeValue(in, classMap);
                    ValueObject val = decodeValue(in, classMap);
                    map.put(key.toString(), val);
                }
                return map;
            case CUSTOM_INDEX:
                Class<? extends ValueObject> ci = classMap.getObject((int) Bytes.readLength(in));
                return rehydrate(ci, in, classMap);
            case CUSTOM_CLASS:
                int index = (int) Bytes.readLength(in);
                Class<? extends ValueObject> cc;
                try {
                    cc = (Class<? extends ValueObject>) Class.forName(Bytes.readString(in));
                } catch (ClassNotFoundException e) {
                    throw new RuntimeException(e);
                }
                if (classMap.setObjectIndex(index, cc) == null) {
                    throw new RuntimeException("index conflict for " + cc + " @ " + index);
                }
                return rehydrate(cc, in, classMap);
            default:
                throw new RuntimeException("invalid decode type " + type);
        }
    }

    /** */
    private static ValueCustom rehydrate(Class<? extends ValueObject> cc, InputStream in, ClassIndexMap classMap) {
        try {
            ValueCustom vc = cc.newInstance().asCustom();
            ValueMap map = decodeValue(in, classMap).asMap();
            vc.setValues(map);
            return vc;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * encode result to a stream
     */
    public static void toOutputStream(DataTable result, OutputStream out) {
        ClassIndexMap classMap = createClassIndexMap();
        FieldIndexMap fieldMap = createFieldIndexMap();

        try {
            Bytes.writeLength(result.size(), out);
            for (Bundle row : result) {
                encodeBundle(row, out, fieldMap, classMap);
            }
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }

    /**
     * encode result to bytes -- helper
     */
    public static byte[] toBytes(DataTable result) {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        toOutputStream(result, out);
        return out.toByteArray();
    }

    /**
     * decode results from bytes -- helper
     */
    public static void fromBytes(DataTable result, byte raw[]) {
        ByteArrayInputStream in = new ByteArrayInputStream(raw);
        fromInputStream(result, in);
    }

    /**
     * decode result from stream
     */
    public static void fromInputStream(DataTable result, InputStream in) {
        ClassIndexMap classMap = createClassIndexMap();
        FieldIndexMap fieldMap = createFieldIndexMap();

        try {
            long rows = Bytes.readLength(in);
            while (rows-- > 0) {
                Bundle row = result.createBundle();
                decodeBundle(row, in, fieldMap, classMap);
                result.append(row);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    private static class OIM<T> implements ObjectIndexMap<T> {

        private final HashMap<Integer, T> indexMap = new HashMap<>();
        private final HashMap<T, Integer> mapIndex = new HashMap<>();

        @Override
        public String toString() {
            return "OIM[" + indexMap.size() + "," + mapIndex.size() + "](im=" + indexMap + ",mi=" + mapIndex + ")";
        }

        @Override
        public void reset() {
            indexMap.clear();
            mapIndex.clear();
        }

        @Override
        public int size() {
            return indexMap.size();
        }

        @Override
        public Integer getObjectIndex(T obj) {
            return mapIndex.get(obj);
        }

        @Override
        public Integer createObjectIndex(T obj) {
            Integer index = mapIndex.get(obj);
            if (index != null) {
                return null;
            }
            index = indexMap.size() + 1;
            indexMap.put(index, obj);
            mapIndex.put(obj, index);
            return index;
        }

        @Override
        public Integer setObjectIndex(int index, T obj) {
            if (indexMap.get(index) != null) {
                return null;
            }
            indexMap.put(index, obj);
            mapIndex.put(obj, index);
            return index;
        }

        @Override
        public T getObject(Integer index) {
            if (index > indexMap.size()) {
                throw new IllegalArgumentException("map index out of bounds : " + index + " > " + indexMap.size());
            }
            return indexMap.get(index);
        }
    }

    /**
     * for stateless mapping
     */
    private static class OIMNull<T> implements ObjectIndexMap<T> {

        @Override
        public void reset() {
        }

        @Override
        public int size() {
            return 0;
        }

        @Override
        public Integer getObjectIndex(T type) {
            return null;
        }

        @Override
        public Integer createObjectIndex(T type) {
            return 0;
        }

        @Override
        public Integer setObjectIndex(int index, T type) {
            return null;
        }

        @Override
        public T getObject(Integer index) {
            return null;
        }
    }

    /** */
    private static final class CIM extends OIM<Class<? extends ValueObject>> implements ClassIndexMap {

    }

    /** */
    private static final class FIM extends OIM<BundleField> implements FieldIndexMap {

    }

    /** */
    private static final class CIMNull extends OIMNull<Class<? extends ValueObject>> implements ClassIndexMap {

    }

    /** */
    private static final class FIMNull extends OIMNull<BundleField> implements FieldIndexMap {

    }
}
TOP

Related Classes of com.addthis.bundle.io.DataChannelCodec$ObjectIndexMap

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.