/**
* Licensed to Odiago, Inc. under one or more contributor license
* agreements. See the NOTICE file distributed with this work for
* additional information regarding copyright ownership. Odiago, Inc.
* licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package org.apache.avro.hadoop.io;
import java.io.IOException;
import org.apache.avro.Schema;
import org.apache.avro.mapred.AvroKey;
import org.apache.avro.mapred.AvroValue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFileBase;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A wrapper around a Hadoop {@link org.apache.hadoop.io.SequenceFile} that
* also supports reading and writing Avro data.
*
* <p>The vanilla Hadoop <code>SequenceFile</code> contains a <i>header</i>
* followed by a sequence of <i>records</i>. A <i>record</i> consists of a
* <i>key</i> and a <i>value</i>. The <i>key</i> and <i>value</i> must
* either:</p>
*
* <ul>
* <li>implement the <code>Writable</code> interface, or</li>
* <li>be accepted by a <code>Serialization</code> registered with the
* <code>SerializationFactory</code>.</li>
* </ul>
*
* <p>Since Avro data are Plain Old Java Objects (e.g., <code>Integer</code>
* for data with schema <i>"int"</i>), they do not implement <i>Writable</i>.
* Furthermore, a {@link org.apache.hadoop.io.Serialization} implementation
* cannot determine whether an object instance of type
* <code>CharSequence</code> that also implements <code>Writable</code> should
* be serialized using Avro or WritableSerialization.</p>
*
* <p>The solution implemented in <code>AvroSequenceFile</code> is to:</p>
*
* <ul>
* <li>wrap Avro key data in an <code>AvroKey</code> object,</li>
* <li>wrap Avro value data in an <code>AvroValue</code> object,</li>
* <li>configure and register <code>AvroSerialization</code> with the
* <code>SerializationFactory</code>, which will accept only objects that are instances
* of either <code>AvroKey</code> or <code>AvroValue</code>, and</li>
* <li>store the Avro key and value schemas in the SequenceFile <i>header</i>.</li>
* </ul>
*/
public class AvroSequenceFile extends SequenceFileBase {
private static final Logger LOG = LoggerFactory.getLogger(AvroSequenceFile.class);
/** The SequencFile.Metadata field for the Avro key writer schema. */
public static final Text METADATA_FIELD_KEY_SCHEMA = new Text("avro.key.schema");
/** The SequencFile.Metadata field for the Avro value writer schema. */
public static final Text METADATA_FIELD_VALUE_SCHEMA = new Text("avro.value.schema");
/** Constructor disabled for this container class. */
private AvroSequenceFile() {}
/**
* Creates a writer from a set of options.
*
* <p>Since there are different implementations of <code>Writer</code> depending on the
* compression type, this method constructs the appropriate subclass depending on the
* compression type given in the <code>options</code>.</p>
*
* @param options The options for the writer.
* @return A new writer instance.
* @throws IOException If the writer cannot be created.
*/
public static SequenceFile.Writer createWriter(Writer.Options options) throws IOException {
switch (options.getCompressionType()) {
case NONE:
return new Writer(options);
case RECORD:
return new RecordCompressWriter(options);
case BLOCK:
return new BlockCompressWriter(options);
default:
throw new IllegalArgumentException(
"Invalid compression type: " + options.getCompressionType());
}
}
/**
* A writer for an uncompressed SequenceFile that supports Avro data.
*/
public static class Writer extends SequenceFile.Writer {
/**
* A helper class to encapsulate the options that can be used to construct a Writer.
*/
public static class Options {
/** The default write buffer size in bytes. */
public static final int DEFAULT_BUFFER_SIZE_BYTES = 4096;
/**
* A magic value representing the default for buffer size, block size, and
* replication factor.
*/
private static final short DEFAULT = -1;
private FileSystem mFileSystem;
private Configuration mConf;
private Path mOutputPath;
private Class<?> mKeyClass;
private Schema mKeyWriterSchema;
private Class<?> mValueClass;
private Schema mValueWriterSchema;
private int mBufferSizeBytes;
private short mReplicationFactor;
private long mBlockSizeBytes;
private Progressable mProgressable;
private CompressionType mCompressionType;
private CompressionCodec mCompressionCodec;
private Metadata mMetadata;
/**
* Creates a new <code>Options</code> instance with default values.
*/
public Options() {
mBufferSizeBytes = DEFAULT;
mReplicationFactor = DEFAULT;
mBlockSizeBytes = DEFAULT;
mCompressionType = CompressionType.NONE;
mMetadata = new Metadata();
}
/**
* Sets the filesystem the SequenceFile should be written to.
*
* @param fileSystem The filesystem.
* @return This options instance.
*/
public Options withFileSystem(FileSystem fileSystem) {
if (null == fileSystem) {
throw new IllegalArgumentException("Filesystem may not be null");
}
mFileSystem = fileSystem;
return this;
}
/**
* Sets the Hadoop configuration.
*
* @param conf The configuration.
* @return This options instance.
*/
public Options withConfiguration(Configuration conf) {
if (null == conf) {
throw new IllegalArgumentException("Configuration may not be null");
}
mConf = conf;
return this;
}
/**
* Sets the output path for the SequenceFile.
*
* @param outputPath The output path.
* @return This options instance.
*/
public Options withOutputPath(Path outputPath) {
if (null == outputPath) {
throw new IllegalArgumentException("Output path may not be null");
}
mOutputPath = outputPath;
return this;
}
/**
* Sets the class of the key records to be written.
*
* <p>If the keys will be Avro data, use {@link
* #withKeySchema(org.apache.avro.Schema)} to specify the writer schema. The key
* class will be automatically set to {@link org.apache.avro.mapred.AvroKey}.</p>
*
* @param keyClass The key class.
* @return This options instance.
*/
public Options withKeyClass(Class<?> keyClass) {
if (null == keyClass) {
throw new IllegalArgumentException("Key class may not be null");
}
mKeyClass = keyClass;
return this;
}
/**
* Sets the writer schema of the key records when using Avro data.
*
* <p>The key class will automatically be set to {@link
* org.apache.avro.mapred.AvroKey}, so there is no need to call {@link
* #withKeyClass(Class)} when using this method.</p>
*
* @param keyWriterSchema The writer schema for the keys.
* @return This options instance.
*/
public Options withKeySchema(Schema keyWriterSchema) {
if (null == keyWriterSchema) {
throw new IllegalArgumentException("Key schema may not be null");
}
withKeyClass(AvroKey.class);
mKeyWriterSchema = keyWriterSchema;
return this;
}
/**
* Sets the class of the value records to be written.
*
* <p>If the values will be Avro data, use {@link
* #withValueSchema(org.apache.avro.Schema)} to specify the writer schema. The value
* class will be automatically set to {@link org.apache.avro.mapred.AvroValue}.</p>
*
* @param valueClass The value class.
* @return This options instance.
*/
public Options withValueClass(Class<?> valueClass) {
if (null == valueClass) {
throw new IllegalArgumentException("Value class may not be null");
}
mValueClass = valueClass;
return this;
}
/**
* Sets the writer schema of the value records when using Avro data.
*
* <p>The value class will automatically be set to {@link
* org.apache.avro.mapred.AvroValue}, so there is no need to call {@link
* #withValueClass(Class)} when using this method.</p>
*
* @param valueWriterSchema The writer schema for the values.
* @return This options instance.
*/
public Options withValueSchema(Schema valueWriterSchema) {
if (null == valueWriterSchema) {
throw new IllegalArgumentException("Value schema may not be null");
}
withValueClass(AvroValue.class);
mValueWriterSchema = valueWriterSchema;
return this;
}
/**
* Sets the write buffer size in bytes.
*
* @param bytes The desired buffer size.
* @return This options instance.
*/
public Options withBufferSizeBytes(int bytes) {
if (bytes < 0) {
throw new IllegalArgumentException("Buffer size may not be negative");
}
mBufferSizeBytes = bytes;
return this;
}
/**
* Sets the desired replication factor for the file.
*
* @param replicationFactor The replication factor.
* @return This options instance.
*/
public Options withReplicationFactor(short replicationFactor) {
if (replicationFactor <= 0) {
throw new IllegalArgumentException("Replication factor must be positive");
}
mReplicationFactor = replicationFactor;
return this;
}
/**
* Sets the desired size of the file blocks.
*
* @param bytes The desired block size in bytes.
* @return This options instance.
*/
public Options withBlockSizeBytes(long bytes) {
if (bytes <= 0) {
throw new IllegalArgumentException("Block size must be positive");
}
mBlockSizeBytes = bytes;
return this;
}
/**
* Sets an object to report progress to.
*
* @param progressable A progressable object to track progress.
* @return This options instance.
*/
public Options withProgressable(Progressable progressable) {
mProgressable = progressable;
return this;
}
/**
* Sets the type of compression.
*
* @param compressionType The type of compression for the output file.
* @return This options instance.
*/
public Options withCompressionType(CompressionType compressionType) {
mCompressionType = compressionType;
return this;
}
/**
* Sets the compression codec to use if it is enabled.
*
* @param compressionCodec The compression codec.
* @return This options instance.
*/
public Options withCompressionCodec(CompressionCodec compressionCodec) {
mCompressionCodec = compressionCodec;
return this;
}
/**
* Sets the metadata that should be stored in the file <i>header</i>.
*
* @param metadata The file metadata.
* @return This options instance.
*/
public Options withMetadata(Metadata metadata) {
if (null == metadata) {
throw new IllegalArgumentException("Metadata may not be null");
}
mMetadata = metadata;
return this;
}
/**
* Gets the filesystem the SequenceFile should be written to.
*
* @return The file system to write to.
*/
public FileSystem getFileSystem() {
if (null == mFileSystem) {
throw new RuntimeException("Must call Options.withFileSystem()");
}
return mFileSystem;
}
/**
* Gets the Hadoop configuration.
*
* @return The Hadoop configuration.
*/
public Configuration getConfiguration() {
return mConf;
}
/**
* Gets the Hadoop configuration with Avro serialization registered.
*
* @return The Hadoop configuration.
*/
public Configuration getConfigurationWithAvroSerialization() {
Configuration conf = getConfiguration();
if (null == conf) {
throw new RuntimeException("Must call Options.withConfiguration()");
}
Configuration confWithAvro = new Configuration(conf);
if (null != mKeyWriterSchema) {
AvroSerialization.setKeyWriterSchema(confWithAvro, mKeyWriterSchema);
}
if (null != mValueWriterSchema) {
AvroSerialization.setValueWriterSchema(confWithAvro, mValueWriterSchema);
}
AvroSerialization.addToConfiguration(confWithAvro);
return confWithAvro;
}
/**
* Gets the output path for the sequence file.
*
* @return The output path.
*/
public Path getOutputPath() {
if (null == mOutputPath) {
throw new RuntimeException("Must call Options.withOutputPath()");
}
return mOutputPath;
}
/**
* Gets the class of the key records.
*
* @return The key class.
*/
public Class<?> getKeyClass() {
if (null == mKeyClass) {
throw new RuntimeException(
"Must call Options.withKeyClass() or Options.withKeySchema()");
}
return mKeyClass;
}
/**
* Gets the class of the value records.
*
* @return The value class.
*/
public Class<?> getValueClass() {
if (null == mValueClass) {
throw new RuntimeException(
"Must call Options.withValueClass() or Options.withValueSchema()");
}
return mValueClass;
}
/**
* Gets the desired size of the buffer used when flushing records to disk.
*
* @return The buffer size in bytes.
*/
public int getBufferSizeBytes() {
if (DEFAULT == mBufferSizeBytes) {
return getConfiguration().getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE_BYTES);
}
return mBufferSizeBytes;
}
/**
* Gets the desired number of replicas to store for each block of the file.
*
* @return The replciation factor for the blocks of the file.
*/
public short getReplicationFactor() {
if (DEFAULT == mReplicationFactor) {
return getFileSystem().getDefaultReplication();
}
return mReplicationFactor;
}
/**
* Gets the desired size of the file blocks.
*
* @return The size of a file block in bytes.
*/
public long getBlockSizeBytes() {
if (DEFAULT == mBlockSizeBytes) {
return getFileSystem().getDefaultBlockSize();
}
return mBlockSizeBytes;
}
/**
* Gets the object to report progress to.
*
* @return A progressable object to track progress.
*/
public Progressable getProgressable() {
return mProgressable;
}
/**
* Gets the type of compression.
*
* @return The compression type.
*/
public CompressionType getCompressionType() {
return mCompressionType;
}
/**
* Gets the compression codec.
*
* @return The compression codec.
*/
public CompressionCodec getCompressionCodec() {
return mCompressionCodec;
}
/**
* Gets the SequenceFile metadata to store in the <i>header</i>.
*
* @return The metadata header.
*/
public Metadata getMetadata() {
return mMetadata;
}
/**
* Gets the metadata to store in the file header, which includes
* any necessary Avro writer schemas.
*
* @return The metadata header with Avro writer schemas if Avro data is being written.
*/
private Metadata getMetadataWithAvroSchemas() {
// mMetadata was intialized in the constructor, and cannot be set to null.
assert null != mMetadata;
if (null != mKeyWriterSchema) {
mMetadata.set(METADATA_FIELD_KEY_SCHEMA, new Text(mKeyWriterSchema.toString()));
}
if (null != mValueWriterSchema) {
mMetadata.set(METADATA_FIELD_VALUE_SCHEMA, new Text(mValueWriterSchema.toString()));
}
return mMetadata;
}
}
/**
* Creates a new <code>Writer</code> to a SequenceFile that supports Avro data.
*
* @param options The writer options.
* @throws IOException If the writer cannot be initialized.
*/
public Writer(Options options) throws IOException {
super(options.getFileSystem(), options.getConfigurationWithAvroSerialization(),
options.getOutputPath(), options.getKeyClass(), options.getValueClass(),
options.getBufferSizeBytes(), options.getReplicationFactor(),
options.getBlockSizeBytes(), options.getProgressable(),
options.getMetadataWithAvroSchemas());
}
}
/**
* A Writer for Avro-enabled SequenceFiles using record-level compression.
*/
public static class RecordCompressWriter extends RecordCompressWriterBase {
/**
* Creates a new <code>RecordCompressWriter</code> to a SequenceFile that supports Avro data.
*
* @param options The writer options.
* @throws IOException If the writer cannot be initialized.
*/
public RecordCompressWriter(Writer.Options options) throws IOException {
super(options.getFileSystem(), options.getConfigurationWithAvroSerialization(),
options.getOutputPath(), options.getKeyClass(), options.getValueClass(),
options.getBufferSizeBytes(), options.getReplicationFactor(),
options.getBlockSizeBytes(), options.getCompressionCodec(),
options.getProgressable(), options.getMetadataWithAvroSchemas());
}
}
/**
* A Writer for Avro-enabled SequenceFiles using block-level compression.
*/
public static class BlockCompressWriter extends BlockCompressWriterBase {
/**
* Creates a new <code>BlockCompressWriter</code> to a SequenceFile that supports Avro data.
*
* @param options The writer options.
* @throws IOException If the writer cannot be initialized.
*/
public BlockCompressWriter(Writer.Options options) throws IOException {
super(options.getFileSystem(), options.getConfigurationWithAvroSerialization(),
options.getOutputPath(), options.getKeyClass(), options.getValueClass(),
options.getBufferSizeBytes(), options.getReplicationFactor(),
options.getBlockSizeBytes(), options.getCompressionCodec(),
options.getProgressable(), options.getMetadataWithAvroSchemas());
}
}
/**
* A reader for SequenceFiles that may contain Avro data.
*/
public static class Reader extends SequenceFile.Reader {
/**
* A helper class to encapsulate the options that can be used to construct a Reader.
*/
public static class Options {
private FileSystem mFileSystem;
private Path mInputPath;
private Configuration mConf;
private Schema mKeyReaderSchema;
private Schema mValueReaderSchema;
/**
* Sets the filesystem the SequenceFile should be read from.
*
* @param fileSystem The filesystem.
* @return This options instance.
*/
public Options withFileSystem(FileSystem fileSystem) {
if (null == fileSystem) {
throw new IllegalArgumentException("Filesystem may not be null");
}
mFileSystem = fileSystem;
return this;
}
/**
* Sets the input path for the SequenceFile.
*
* @param inputPath The input path.
* @return This options instance.
*/
public Options withInputPath(Path inputPath) {
if (null == inputPath) {
throw new IllegalArgumentException("Input path may not be null");
}
mInputPath = inputPath;
return this;
}
/**
* Sets the Hadoop configuration.
*
* @param conf The configuration.
* @return This options instance.
*/
public Options withConfiguration(Configuration conf) {
if (null == conf) {
throw new IllegalArgumentException("Configuration may not be null");
}
mConf = conf;
return this;
}
/**
* Sets the reader schema of the key records when using Avro data.
*
* <p>If not set, the writer schema will be used as the reader schema.</p>
*
* @param keyReaderSchema The reader schema for the keys.
* @return This options instance.
*/
public Options withKeySchema(Schema keyReaderSchema) {
mKeyReaderSchema = keyReaderSchema;
return this;
}
/**
* Sets the reader schema of the value records when using Avro data.
*
* <p>If not set, the writer schema will be used as the reader schema.</p>
*
* @param valueReaderSchema The reader schema for the values.
* @return This options instance.
*/
public Options withValueSchema(Schema valueReaderSchema) {
mValueReaderSchema = valueReaderSchema;
return this;
}
/**
* Gets the filesystem the SequenceFile should be read rom.
*
* @return The file system to read from.
*/
public FileSystem getFileSystem() {
if (null == mFileSystem) {
throw new RuntimeException("Must call Options.withFileSystem()");
}
return mFileSystem;
}
/**
* Gets the input path for the sequence file.
*
* @return The input path.
*/
public Path getInputPath() {
if (null == mInputPath) {
throw new RuntimeException("Must call Options.withInputPath()");
}
return mInputPath;
}
/**
* Gets the Hadoop configuration.
*
* @return The Hadoop configuration.
*/
public Configuration getConfiguration() {
return mConf;
}
/**
* Gets the Hadoop configuration with Avro serialization registered.
*
* @return The Hadoop configuration.
* @throws IOException If there is an error configuring Avro serialization.
*/
public Configuration getConfigurationWithAvroSerialization() throws IOException {
Configuration conf = getConfiguration();
if (null == conf) {
throw new RuntimeException("Must call Options.withConfiguration()");
}
// Configure schemas and add Avro serialization to the configuration.
Configuration confWithAvro = new Configuration(conf);
AvroSerialization.addToConfiguration(confWithAvro);
// Read the metadata header from the SequenceFile to get the writer schemas.
Metadata metadata = AvroSequenceFile.getMetadata(
getFileSystem(), getInputPath(), confWithAvro);
// Set the key schema if present in the metadata.
Text keySchemaText = metadata.get(METADATA_FIELD_KEY_SCHEMA);
if (null != keySchemaText) {
LOG.debug("Using key writer schema from SequenceFile metadata: "
+ keySchemaText.toString());
AvroSerialization.setKeyWriterSchema(
confWithAvro, Schema.parse(keySchemaText.toString()));
if (null != mKeyReaderSchema) {
AvroSerialization.setKeyReaderSchema(confWithAvro, mKeyReaderSchema);
}
}
// Set the value schema if present in the metadata.
Text valueSchemaText = metadata.get(METADATA_FIELD_VALUE_SCHEMA);
if (null != valueSchemaText) {
LOG.debug("Using value writer schema from SequenceFile metadata: "
+ valueSchemaText.toString());
AvroSerialization.setValueWriterSchema(
confWithAvro, Schema.parse(valueSchemaText.toString()));
if (null != mValueReaderSchema) {
AvroSerialization.setValueReaderSchema(confWithAvro, mValueReaderSchema);
}
}
return confWithAvro;
}
}
/**
* Creates a new <code>Reader</code> from a SequenceFile that supports Avro data.
*
* @param options The reader options.
* @throws IOException If the reader cannot be initialized.
*/
public Reader(Options options) throws IOException {
super(options.getFileSystem(), options.getInputPath(),
options.getConfigurationWithAvroSerialization());
}
}
/**
* Open and read just the metadata header from a SequenceFile.
*
* @param fs The FileSystem the SequenceFile is on.
* @param path The path to the file.
* @param conf The Hadoop configuration.
* @return The metadata header.
* @throws IOException If the metadata cannot be read from the file.
*/
private static Metadata getMetadata(FileSystem fs, Path path, Configuration conf)
throws IOException {
SequenceFile.Reader metadataReader = null;
try {
metadataReader = new SequenceFile.Reader(fs, path, conf);
return metadataReader.getMetadata();
} finally {
if (null != metadataReader) {
metadataReader.close();
}
}
}
}