/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.giraph.utils;
import static org.apache.hadoop.util.ReflectionUtils.newInstance;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.OutEdges;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.zk.ZooKeeperExt;
import org.apache.giraph.zk.ZooKeeperExt.PathStat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
/**
* Helper static methods for working with Writable objects.
*/
public class WritableUtils {
/**
* Don't construct.
*/
private WritableUtils() { }
/**
* Instantiate a new Writable, checking for NullWritable along the way.
*
* @param klass Class
* @param <W> type
* @return new instance of class
*/
public static <W extends Writable> W createWritable(Class<W> klass) {
return createWritable(klass, null);
}
/**
* Instantiate a new Writable, checking for NullWritable along the way.
*
* @param klass Class
* @param configuration Configuration
* @param <W> type
* @return new instance of class
*/
public static <W extends Writable> W createWritable(
Class<W> klass,
ImmutableClassesGiraphConfiguration configuration) {
W result;
if (NullWritable.class.equals(klass)) {
result = (W) NullWritable.get();
} else {
result = ReflectionUtils.newInstance(klass);
}
ConfigurationUtils.configureIfPossible(result, configuration);
return result;
}
/**
* Read fields from byteArray to a Writeable object.
*
* @param byteArray Byte array to find the fields in.
* @param writableObjects Objects to fill in the fields.
*/
public static void readFieldsFromByteArray(
byte[] byteArray, Writable... writableObjects) {
DataInputStream inputStream =
new DataInputStream(new ByteArrayInputStream(byteArray));
try {
for (Writable writableObject : writableObjects) {
writableObject.readFields(inputStream);
}
} catch (IOException e) {
throw new IllegalStateException(
"readFieldsFromByteArray: IOException", e);
}
}
/**
* Read fields from a ZooKeeper znode.
*
* @param zkExt ZooKeeper instance.
* @param zkPath Path of znode.
* @param watch Add a watch?
* @param stat Stat of znode if desired.
* @param writableObjects Objects to read into.
*/
public static void readFieldsFromZnode(ZooKeeperExt zkExt,
String zkPath,
boolean watch,
Stat stat,
Writable... writableObjects) {
try {
byte[] zkData = zkExt.getData(zkPath, false, stat);
readFieldsFromByteArray(zkData, writableObjects);
} catch (KeeperException e) {
throw new IllegalStateException(
"readFieldsFromZnode: KeeperException on " + zkPath, e);
} catch (InterruptedException e) {
throw new IllegalStateException(
"readFieldsFromZnode: InterrruptedStateException on " + zkPath, e);
}
}
/**
* Write object to a byte array.
*
* @param writableObjects Objects to write from.
* @return Byte array with serialized object.
*/
public static byte[] writeToByteArray(Writable... writableObjects) {
ByteArrayOutputStream outputStream =
new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outputStream);
try {
for (Writable writableObject : writableObjects) {
writableObject.write(output);
}
} catch (IOException e) {
throw new IllegalStateException(
"writeToByteArray: IOStateException", e);
}
return outputStream.toByteArray();
}
/**
* Read fields from byteArray to a Writeable object, skipping the size.
* Serialization method is choosable
*
* @param byteArray Byte array to find the fields in.
* @param writableObject Object to fill in the fields.
* @param unsafe Use unsafe deserialization
*/
public static void readFieldsFromByteArrayWithSize(
byte[] byteArray, Writable writableObject, boolean unsafe) {
ExtendedDataInput extendedDataInput;
if (unsafe) {
extendedDataInput = new UnsafeByteArrayInputStream(byteArray);
} else {
extendedDataInput = new ExtendedByteArrayDataInput(byteArray);
}
try {
extendedDataInput.readInt();
writableObject.readFields(extendedDataInput);
} catch (IOException e) {
throw new IllegalStateException(
"readFieldsFromByteArrayWithSize: IOException", e);
}
}
/**
* Write object to a byte array with the first 4 bytes as the size of the
* entire buffer (including the size).
*
* @param writableObject Object to write from.
* @param unsafe Use unsafe serialization?
* @return Byte array with serialized object.
*/
public static byte[] writeToByteArrayWithSize(Writable writableObject,
boolean unsafe) {
return writeToByteArrayWithSize(writableObject, null, unsafe);
}
/**
* Write object to a byte array with the first 4 bytes as the size of the
* entire buffer (including the size).
*
* @param writableObject Object to write from.
* @param buffer Use this buffer instead
* @param unsafe Use unsafe serialization?
* @return Byte array with serialized object.
*/
public static byte[] writeToByteArrayWithSize(Writable writableObject,
byte[] buffer,
boolean unsafe) {
ExtendedDataOutput extendedDataOutput;
if (unsafe) {
extendedDataOutput = new UnsafeByteArrayOutputStream(buffer);
} else {
extendedDataOutput = new ExtendedByteArrayDataOutput(buffer);
}
try {
extendedDataOutput.writeInt(-1);
writableObject.write(extendedDataOutput);
extendedDataOutput.writeInt(0, extendedDataOutput.getPos());
} catch (IOException e) {
throw new IllegalStateException("writeToByteArrayWithSize: " +
"IOException", e);
}
return extendedDataOutput.getByteArray();
}
/**
* Write object to a ZooKeeper znode.
*
* @param zkExt ZooKeeper instance.
* @param zkPath Path of znode.
* @param version Version of the write.
* @param writableObjects Objects to write from.
* @return Path and stat information of the znode.
*/
public static PathStat writeToZnode(ZooKeeperExt zkExt,
String zkPath,
int version,
Writable... writableObjects) {
try {
byte[] byteArray = writeToByteArray(writableObjects);
return zkExt.createOrSetExt(zkPath,
byteArray,
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT,
true,
version);
} catch (KeeperException e) {
throw new IllegalStateException(
"writeToZnode: KeeperException on " + zkPath, e);
} catch (InterruptedException e) {
throw new IllegalStateException(
"writeToZnode: InterruptedException on " + zkPath, e);
}
}
/**
* Write list of object to a byte array.
*
* @param writableList List of object to write from.
* @return Byte array with serialized objects.
*/
public static byte[] writeListToByteArray(
List<? extends Writable> writableList) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(outputStream);
try {
output.writeInt(writableList.size());
for (Writable writable : writableList) {
writable.write(output);
}
} catch (IOException e) {
throw new IllegalStateException(
"writeListToByteArray: IOException", e);
}
return outputStream.toByteArray();
}
/**
* Write list of objects to a ZooKeeper znode.
*
* @param zkExt ZooKeeper instance.
* @param zkPath Path of znode.
* @param version Version of the write.
* @param writableList List of objects to write from.
* @return Path and stat information of the znode.
*/
public static PathStat writeListToZnode(
ZooKeeperExt zkExt,
String zkPath,
int version,
List<? extends Writable> writableList) {
try {
return zkExt.createOrSetExt(
zkPath,
writeListToByteArray(writableList),
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT,
true,
version);
} catch (KeeperException e) {
throw new IllegalStateException(
"writeListToZnode: KeeperException on " + zkPath, e);
} catch (InterruptedException e) {
throw new IllegalStateException(
"writeListToZnode: InterruptedException on " + zkPath, e);
}
}
/**
* Read fields from byteArray to a list of objects.
*
* @param byteArray Byte array to find the fields in.
* @param writableClass Class of the objects to instantiate.
* @param conf Configuration used for instantiation (i.e Configurable)
* @param <T> Object type
* @return List of objects.
*/
public static <T extends Writable> List<T> readListFieldsFromByteArray(
byte[] byteArray,
Class<? extends T> writableClass,
Configuration conf) {
try {
DataInputStream inputStream =
new DataInputStream(new ByteArrayInputStream(byteArray));
int size = inputStream.readInt();
List<T> writableList = new ArrayList<T>(size);
for (int i = 0; i < size; ++i) {
T writable = newInstance(writableClass, conf);
writable.readFields(inputStream);
writableList.add(writable);
}
return writableList;
} catch (IOException e) {
throw new IllegalStateException(
"readListFieldsFromZnode: IOException", e);
}
}
/**
* Read fields from a ZooKeeper znode into a list of objects.
*
* @param zkExt ZooKeeper instance.
* @param zkPath Path of znode.
* @param watch Add a watch?
* @param stat Stat of znode if desired.
* @param writableClass Class of the objects to instantiate.
* @param conf Configuration used for instantiation (i.e Configurable)
* @param <T> Object type
* @return List of objects.
*/
public static <T extends Writable> List<T> readListFieldsFromZnode(
ZooKeeperExt zkExt,
String zkPath,
boolean watch,
Stat stat,
Class<? extends T> writableClass,
Configuration conf) {
try {
byte[] zkData = zkExt.getData(zkPath, false, stat);
return WritableUtils.<T>readListFieldsFromByteArray(zkData,
writableClass, conf);
} catch (KeeperException e) {
throw new IllegalStateException(
"readListFieldsFromZnode: KeeperException on " + zkPath, e);
} catch (InterruptedException e) {
throw new IllegalStateException(
"readListFieldsFromZnode: InterruptedException on " + zkPath,
e);
}
}
/**
* Write ExtendedDataOutput to DataOutput
*
* @param extendedDataOutput ExtendedDataOutput to write
* @param out DataOutput to write to
*/
public static void writeExtendedDataOutput(
ExtendedDataOutput extendedDataOutput, DataOutput out)
throws IOException {
out.writeInt(extendedDataOutput.getPos());
out.write(
extendedDataOutput.getByteArray(), 0, extendedDataOutput.getPos());
}
/**
* Read ExtendedDataOutput from DataInput
*
* @param in DataInput to read from
* @param conf Configuration
* @return ExtendedDataOutput read
*/
public static ExtendedDataOutput readExtendedDataOutput(DataInput in,
ImmutableClassesGiraphConfiguration conf) throws IOException {
int size = in.readInt();
byte[] buf = new byte[size];
in.readFully(buf);
return conf.createExtendedDataOutput(buf, size);
}
/**
* Write vertex data to byte array with the first 4 bytes as the size of the
* entire buffer (including the size).
*
* @param vertex Vertex to write from.
* @param buffer Use this buffer instead
* @param unsafe Use unsafe serialization?
* @param conf Configuration
* @param <I> Vertex id
* @param <V> Vertex value
* @param <E> Edge value
* @return Byte array with serialized object.
*/
public static <I extends WritableComparable, V extends Writable,
E extends Writable> byte[] writeVertexToByteArray(
Vertex<I, V, E> vertex,
byte[] buffer,
boolean unsafe,
ImmutableClassesGiraphConfiguration<I, V, E> conf) {
ExtendedDataOutput extendedDataOutput;
if (unsafe) {
extendedDataOutput = new UnsafeByteArrayOutputStream(buffer);
} else {
extendedDataOutput = new ExtendedByteArrayDataOutput(buffer);
}
try {
extendedDataOutput.writeInt(-1);
writeVertexToDataOutput(extendedDataOutput, vertex, conf);
extendedDataOutput.writeInt(0, extendedDataOutput.getPos());
} catch (IOException e) {
throw new IllegalStateException("writeVertexToByteArray: " +
"IOException", e);
}
return extendedDataOutput.getByteArray();
}
/**
* Write vertex data to byte array with the first 4 bytes as the size of the
* entire buffer (including the size).
*
* @param vertex Vertex to write from.
* @param unsafe Use unsafe serialization?
* @param conf Configuration
* @param <I> Vertex id
* @param <V> Vertex value
* @param <E> Edge value
* @return Byte array with serialized object.
*/
public static <I extends WritableComparable, V extends Writable,
E extends Writable> byte[] writeVertexToByteArray(
Vertex<I, V, E> vertex,
boolean unsafe,
ImmutableClassesGiraphConfiguration<I, V, E> conf) {
return writeVertexToByteArray(vertex, null, unsafe, conf);
}
/**
* Read vertex data from byteArray to a Writeable object, skipping the size.
* Serialization method is choosable. Assumes the vertex has already been
* initialized and contains values for Id, value, and edges.
*
* @param byteArray Byte array to find the fields in.
* @param vertex Vertex to fill in the fields.
* @param unsafe Use unsafe deserialization
* @param <I> Vertex id
* @param <V> Vertex value
* @param <E> Edge value
* @param conf Configuration
*/
public static <I extends WritableComparable, V extends Writable,
E extends Writable> void reinitializeVertexFromByteArray(
byte[] byteArray,
Vertex<I, V, E> vertex,
boolean unsafe,
ImmutableClassesGiraphConfiguration<I, V, E> conf) {
ExtendedDataInput extendedDataInput;
if (unsafe) {
extendedDataInput = new UnsafeByteArrayInputStream(byteArray);
} else {
extendedDataInput = new ExtendedByteArrayDataInput(byteArray);
}
try {
extendedDataInput.readInt();
reinitializeVertexFromDataInput(extendedDataInput, vertex, conf);
} catch (IOException e) {
throw new IllegalStateException(
"readFieldsFromByteArrayWithSize: IOException", e);
}
}
/**
* Write an edge to an output stream.
*
* @param out Data output
* @param edge Edge to write
* @param <I> Vertex id
* @param <E> Edge value
* @throws IOException
*/
public static <I extends WritableComparable, E extends Writable>
void writeEdge(DataOutput out, Edge<I, E> edge) throws IOException {
edge.getTargetVertexId().write(out);
edge.getValue().write(out);
}
/**
* Read an edge from an input stream.
*
* @param in Data input
* @param edge Edge to fill in-place
* @param <I> Vertex id
* @param <E> Edge value
* @throws IOException
*/
public static <I extends WritableComparable, E extends Writable>
void readEdge(DataInput in, Edge<I, E> edge) throws IOException {
edge.getTargetVertexId().readFields(in);
edge.getValue().readFields(in);
}
/**
* Reads data from input stream to inizialize Vertex. Assumes the vertex has
* already been initialized and contains values for Id, value, and edges.
*
* @param input The input stream
* @param vertex The vertex to initialize
* @param conf Configuration
* @param <I> Vertex id
* @param <V> Vertex value
* @param <E> Edge value
* @throws IOException
*/
@SuppressWarnings("unchecked")
public static <I extends WritableComparable, V extends Writable,
E extends Writable> void reinitializeVertexFromDataInput(
DataInput input,
Vertex<I, V, E> vertex,
ImmutableClassesGiraphConfiguration<I, V, E> conf)
throws IOException {
vertex.getId().readFields(input);
vertex.getValue().readFields(input);
((OutEdges<I, E>) vertex.getEdges()).readFields(input);
if (input.readBoolean()) {
vertex.voteToHalt();
} else {
vertex.wakeUp();
}
}
/**
* Reads data from input stream to initialize Vertex.
*
* @param input The input stream
* @param conf Configuration
* @param <I> Vertex id
* @param <V> Vertex value
* @param <E> Edge value
* @return The vertex
* @throws IOException
*/
public static <I extends WritableComparable, V extends Writable,
E extends Writable> Vertex<I, V, E>
readVertexFromDataInput(
DataInput input,
ImmutableClassesGiraphConfiguration<I, V, E> conf)
throws IOException {
Vertex<I, V, E> vertex = conf.createVertex();
I id = conf.createVertexId();
V value = conf.createVertexValue();
OutEdges<I, E> edges = conf.createOutEdges();
vertex.initialize(id, value, edges);
reinitializeVertexFromDataInput(input, vertex, conf);
return vertex;
}
/**
* Writes Vertex data to output stream.
*
* @param output the output stream
* @param vertex The vertex to serialize
* @param conf Configuration
* @param <I> Vertex id
* @param <V> Vertex value
* @param <E> Edge value
* @throws IOException
*/
@SuppressWarnings("unchecked")
public static <I extends WritableComparable, V extends Writable,
E extends Writable> void writeVertexToDataOutput(
DataOutput output,
Vertex<I, V, E> vertex,
ImmutableClassesGiraphConfiguration<I, V, E> conf)
throws IOException {
vertex.getId().write(output);
vertex.getValue().write(output);
((OutEdges<I, E>) vertex.getEdges()).write(output);
output.writeBoolean(vertex.isHalted());
}
/**
* Write class to data output. Also handles the case when class is null.
*
* @param clazz Class
* @param output Data output
* @param <T> Class type
*/
public static <T> void writeClass(Class<T> clazz,
DataOutput output) throws IOException {
output.writeBoolean(clazz != null);
if (clazz != null) {
output.writeUTF(clazz.getName());
}
}
/**
* Read class from data input.
* Matches {@link #writeClass(Class, DataOutput)}.
*
* @param input Data input
* @param <T> Class type
* @return Class, or null if null was written
*/
@SuppressWarnings("unchecked")
public static <T> Class<T> readClass(DataInput input) throws IOException {
if (input.readBoolean()) {
String className = input.readUTF();
try {
return (Class<T>) Class.forName(className);
} catch (ClassNotFoundException e) {
throw new IllegalStateException("readClass: No class found " +
className);
}
} else {
return null;
}
}
/**
* Write object to output stream
* @param object Object
* @param output Output stream
* @throws IOException
*/
public static void writeWritableObject(
Writable object, DataOutput output)
throws IOException {
output.writeBoolean(object != null);
if (object != null) {
output.writeUTF(object.getClass().getName());
object.write(output);
}
}
/**
* Reads object from input stream
* @param input Input stream
* @param conf Configuration
* @param <T> Object type
* @return Object
* @throws IOException
*/
public static <T extends Writable>
T readWritableObject(DataInput input,
ImmutableClassesGiraphConfiguration conf) throws IOException {
if (input.readBoolean()) {
String className = input.readUTF();
try {
T object =
(T) ReflectionUtils.newInstance(Class.forName(className), conf);
object.readFields(input);
return object;
} catch (ClassNotFoundException e) {
throw new IllegalStateException("readWritableObject: No class found " +
className);
}
} else {
return null;
}
}
/**
* Writes a list of Writable objects into output stream.
* This method is trying to optimize space occupied by class information only
* storing class object if it is different from the previous one
* as in most cases arrays tend to have objects of the same type inside.
* @param list serialized object
* @param output the output stream
* @throws IOException
*/
public static void writeList(List<? extends Writable> list, DataOutput output)
throws IOException {
output.writeBoolean(list != null);
if (list != null) {
output.writeInt(list.size());
Class<? extends Writable> clazz = null;
for (Writable element : list) {
output.writeBoolean(element == null);
if (element != null) {
if (element.getClass() != clazz) {
clazz = element.getClass();
output.writeBoolean(true);
writeClass(clazz, output);
} else {
output.writeBoolean(false);
}
element.write(output);
}
}
}
}
/**
* Reads list of Writable objects from data input stream.
* Input stream should have class information along with object data.
* @param input input stream
* @return deserialized list
* @throws IOException
*/
public static List<? extends Writable> readList(DataInput input)
throws IOException {
try {
List<Writable> res = null;
if (input.readBoolean()) {
int size = input.readInt();
res = new ArrayList<>(size);
Class<? extends Writable> clazz = null;
for (int i = 0; i < size; i++) {
boolean isNull = input.readBoolean();
if (isNull) {
res.add(null);
} else {
boolean hasClassInfo = input.readBoolean();
if (hasClassInfo) {
clazz = readClass(input);
}
Writable element = clazz.newInstance();
element.readFields(input);
res.add(element);
}
}
}
return res;
} catch (InstantiationException | IllegalAccessException e) {
throw new IllegalStateException("unable to instantiate object", e);
}
}
/**
* Create a copy of Writable object, by serializing and deserializing it.
*
* @param reusableOut Reusable output stream to serialize into
* @param reusableIn Reusable input stream to deserialize out of
* @param original Original value of which to make a copy
* @param conf Configuration
* @param <T> Type of the object
* @return Copy of the original value
*/
public static <T extends Writable> T createCopy(
UnsafeByteArrayOutputStream reusableOut,
UnsafeReusableByteArrayInput reusableIn, T original,
ImmutableClassesGiraphConfiguration conf) {
T copy = (T) createWritable(original.getClass(), conf);
try {
reusableOut.reset();
original.write(reusableOut);
reusableIn.initialize(
reusableOut.getByteArray(), 0, reusableOut.getPos());
copy.readFields(reusableIn);
if (reusableIn.available() != 0) {
throw new RuntimeException("Serialization of " +
original.getClass() + " encountered issues, " +
reusableIn.available() + " bytes left to be read");
}
} catch (IOException e) {
throw new IllegalStateException(
"IOException occurred while trying to create a copy " +
original.getClass(), e);
}
return copy;
}
/**
* Writes primitive int array of ints into output stream.
* Array can be null or empty.
* @param array array to be written
* @param dataOutput output stream
* @throws IOException
*/
public static void writeIntArray(int[] array, DataOutput dataOutput)
throws IOException {
if (array != null) {
dataOutput.writeInt(array.length);
for (int r : array) {
dataOutput.writeInt(r);
}
} else {
dataOutput.writeInt(-1);
}
}
/**
* Reads primitive int array from input stream.
* @param dataInput input stream to read from
* @return may return null or empty array.
* @throws IOException
*/
public static int[] readIntArray(DataInput dataInput)
throws IOException {
int [] res = null;
int size = dataInput.readInt();
if (size >= 0) {
res = new int[size];
for (int i = 0; i < size; i++) {
res[i] = dataInput.readInt();
}
}
return res;
}
/**
* Writes primitive long array of ints into output stream.
* Array can be null or empty.
* @param array array to be written
* @param dataOutput output stream
* @throws IOException
*/
public static void writeLongArray(DataOutput dataOutput, long[] array)
throws IOException {
if (array != null) {
dataOutput.writeInt(array.length);
for (long r : array) {
dataOutput.writeLong(r);
}
} else {
dataOutput.writeInt(-1);
}
}
/**
* Reads primitive long array from input stream.
* @param dataInput input stream to read from
* @return may return null or empty array.
* @throws IOException
*/
public static long[] readLongArray(DataInput dataInput)
throws IOException {
long [] res = null;
int size = dataInput.readInt();
if (size >= 0) {
res = new long[size];
for (int i = 0; i < size; i++) {
res[i] = dataInput.readLong();
}
}
return res;
}
}