package org.apache.blur.mapreduce.lib;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.reflect.Constructor;
import org.apache.blur.thirdparty.thrift_0_9_0.TException;
import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol;
import org.apache.blur.thirdparty.thrift_0_9_0.transport.TIOStreamTransport;
import org.apache.blur.thrift.BlurClient;
import org.apache.blur.thrift.generated.Blur.Iface;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.util.Progressable;
/**
* {@link BlurOutputFormat} is used to index data and delivery the indexes to
* the proper Blur table for searching. A typical usage of this class would be
* as follows.<br/>
* <br/>
*
* <br/>
* {@link Iface} client = {@link BlurClient}.getClient("controller1:40010");<br/>
* <br/>
* TableDescriptor tableDescriptor = client.describe(tableName);<br/>
* <br/>
* Job job = new Job(jobConf, "blur index");<br/>
* job.setJarByClass(BlurOutputFormatTest.class);<br/>
* job.setMapperClass(CsvBlurMapper.class);<br/>
* job.setInputFormatClass(TextInputFormat.class);<br/>
* <br/>
* FileInputFormat.addInputPath(job, new Path(input));<br/>
* CsvBlurMapper.addColumns(job, "cf1", "col");<br/>
* <br/>
* BlurOutputFormat.setupJob(job, tableDescriptor);<br/>
* BlurOutputFormat.setIndexLocally(job, true);<br/>
* BlurOutputFormat.setOptimizeInFlight(job, false);<br/>
* <br/>
* job.waitForCompletion(true);<br/>
*
*/
public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> {
public static final String BLUR_OUTPUT_REDUCER_MULTIPLIER = "blur.output.reducer.multiplier";
public static final String BLUR_OUTPUT_OPTIMIZEINFLIGHT = "blur.output.optimizeinflight";
public static final String BLUR_OUTPUT_INDEXLOCALLY = "blur.output.indexlocally";
public static final String BLUR_OUTPUT_MAX_DOCUMENT_BUFFER_SIZE = "blur.output.max.document.buffer.size";
public static final String BLUR_OUTPUT_MAX_DOCUMENT_BUFFER_HEAP_SIZE = "blur.output.max.document.buffer.heap.size";
public static final String BLUR_OUTPUT_DOCUMENT_BUFFER_STRATEGY = "blur.output.document.buffer.strategy";
public static final String BLUR_TABLE_DESCRIPTOR = "blur.table.descriptor";
public static final String BLUR_OUTPUT_PATH = "blur.output.path";
private static final String MAPRED_OUTPUT_COMMITTER_CLASS = "mapred.output.committer.class";
private static ThreadLocal<Progressable> _progressable = new ThreadLocal<Progressable>();
private static ThreadLocal<GetCounter> _getCounter = new ThreadLocal<GetCounter>();
public static void setProgressable(Progressable progressable) {
_progressable.set(progressable);
}
public static Progressable getProgressable() {
return _progressable.get();
}
public static void setGetCounter(GetCounter getCounter) {
_getCounter.set(getCounter);
}
public static GetCounter getGetCounter() {
return _getCounter.get();
}
@Override
public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
CheckOutputSpecs.checkOutputSpecs(context.getConfiguration(), context.getNumReduceTasks());
}
@Override
public RecordWriter<Text, BlurMutate> getRecordWriter(TaskAttemptContext context) throws IOException,
InterruptedException {
int id = context.getTaskAttemptID().getTaskID().getId();
TaskAttemptID taskAttemptID = context.getTaskAttemptID();
final GenericBlurRecordWriter writer = new GenericBlurRecordWriter(context.getConfiguration(), id,
taskAttemptID.toString() + ".tmp");
return new RecordWriter<Text, BlurMutate>() {
@Override
public void write(Text key, BlurMutate value) throws IOException, InterruptedException {
writer.write(key, value);
}
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
writer.close();
}
};
}
@Override
public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
return new BlurOutputCommitter();
}
public static TableDescriptor getTableDescriptor(Configuration configuration) throws IOException {
String tableDesStr = configuration.get(BLUR_TABLE_DESCRIPTOR);
if (tableDesStr == null) {
return null;
}
ByteArrayInputStream inputStream = new ByteArrayInputStream(tableDesStr.getBytes());
TIOStreamTransport transport = new TIOStreamTransport(inputStream);
TJSONProtocol protocol = new TJSONProtocol(transport);
TableDescriptor descriptor = new TableDescriptor();
try {
descriptor.read(protocol);
} catch (TException e) {
throw new IOException(e);
}
transport.close();
return descriptor;
}
/**
* This will multiple the number of reducers for this job. For example if the
* table has 256 shards the normal number of reducers is 256. However if the
* reducer multiplier is set to 4 then the number of reducers will be 1024 and
* each shard will get 4 new segments instead of the normal 1.
*
* @param job
* the job to setup.
* @param multiple
* the multiple to use.
* @throws IOException
*/
public static void setReducerMultiplier(Job job, int multiple) throws IOException {
TableDescriptor tableDescriptor = getTableDescriptor(job.getConfiguration());
if (tableDescriptor == null) {
throw new IOException("setTableDescriptor needs to be called first.");
}
job.setNumReduceTasks(tableDescriptor.getShardCount() * multiple);
Configuration configuration = job.getConfiguration();
configuration.setInt(BLUR_OUTPUT_REDUCER_MULTIPLIER, multiple);
}
public static int getReducerMultiplier(Configuration configuration) {
return configuration.getInt(BLUR_OUTPUT_REDUCER_MULTIPLIER, 1);
}
/**
* Sets the {@link TableDescriptor} for this job.
*
* @param job
* the job to setup.
* @param tableDescriptor
* the {@link TableDescriptor}.
* @throws IOException
*/
public static void setTableDescriptor(Job job, TableDescriptor tableDescriptor) throws IOException {
setTableDescriptor(job.getConfiguration(), tableDescriptor);
}
/**
* Sets the {@link TableDescriptor} for this job.
*
* @param job
* the job to setup.
* @param tableDescriptor
* the {@link TableDescriptor}.
* @throws IOException
*/
public static void setTableDescriptor(Configuration configuration, TableDescriptor tableDescriptor)
throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
TIOStreamTransport transport = new TIOStreamTransport(outputStream);
TJSONProtocol protocol = new TJSONProtocol(transport);
try {
tableDescriptor.write(protocol);
} catch (TException e) {
throw new IOException(e);
}
transport.close();
configuration.set(BLUR_TABLE_DESCRIPTOR, new String(outputStream.toByteArray()));
setOutputPath(configuration, new Path(tableDescriptor.getTableUri()));
}
/**
* Sets the maximum number of documents that the buffer will hold in memory
* before overflowing to disk. By default this is 1000 which will probably be
* very low for most systems.
*
* @param job
* the job to setup.
* @param maxDocumentBufferSize
* the maxDocumentBufferSize.
*/
public static void setMaxDocumentBufferSize(Job job, int maxDocumentBufferSize) {
setMaxDocumentBufferSize(job.getConfiguration(), maxDocumentBufferSize);
}
/**
* Sets the maximum number of documents that the buffer will hold in memory
* before overflowing to disk. By default this is 1000 which will probably be
* very low for most systems.
*
* @param configuration
* the configuration to setup.
* @param maxDocumentBufferSize
* the maxDocumentBufferSize.
*/
public static void setMaxDocumentBufferSize(Configuration configuration, int maxDocumentBufferSize) {
configuration.setInt(BLUR_OUTPUT_MAX_DOCUMENT_BUFFER_SIZE, maxDocumentBufferSize);
}
public static int getMaxDocumentBufferSize(Configuration configuration) {
return configuration.getInt(BLUR_OUTPUT_MAX_DOCUMENT_BUFFER_SIZE, 1000);
}
public static int getMaxDocumentBufferHeapSize(Configuration configuration) {
return configuration.getInt(BLUR_OUTPUT_MAX_DOCUMENT_BUFFER_HEAP_SIZE, 32 * 1024 * 1024);
}
public static void setMaxDocumentBufferHeapSize(Configuration configuration, int maxDocumentBufferHeapSize) {
configuration.setInt(BLUR_OUTPUT_MAX_DOCUMENT_BUFFER_HEAP_SIZE, maxDocumentBufferHeapSize);
}
public static void setMaxDocumentBufferHeapSize(Job job, int maxDocumentBufferHeapSize) {
setMaxDocumentBufferHeapSize(job.getConfiguration(), maxDocumentBufferHeapSize);
}
public static DocumentBufferStrategy getDocumentBufferStrategy(Configuration configuration) {
Class<? extends DocumentBufferStrategy> clazz = configuration.getClass(BLUR_OUTPUT_DOCUMENT_BUFFER_STRATEGY, DocumentBufferStrategyFixedSize.class, DocumentBufferStrategy.class);
try {
Constructor<? extends DocumentBufferStrategy> constructor = clazz.getConstructor(new Class[]{Configuration.class});
return constructor.newInstance(new Object[]{configuration});
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static void setDocumentBufferStrategy(Job job, Class<? extends DocumentBufferStrategy> documentBufferStrategyClass) {
setDocumentBufferStrategy(job.getConfiguration(), documentBufferStrategyClass);
}
public static void setDocumentBufferStrategy(Configuration configuration, Class<? extends DocumentBufferStrategy> documentBufferStrategyClass) {
configuration.setClass(BLUR_OUTPUT_DOCUMENT_BUFFER_STRATEGY, documentBufferStrategyClass, DocumentBufferStrategy.class);
}
public static void setOutputPath(Job job, Path path) {
setOutputPath(job.getConfiguration(), path);
}
public static void setOutputPath(Configuration configuration, Path path) {
configuration.set(BLUR_OUTPUT_PATH, path.toString());
configuration.set(MAPRED_OUTPUT_COMMITTER_CLASS, BlurOutputCommitter.class.getName());
}
public static Path getOutputPath(Configuration configuration) {
return new Path(configuration.get(BLUR_OUTPUT_PATH));
}
/**
* Enabled by default, this will enable local indexing on the machine where
* the task is running. Then when the {@link RecordWriter} closes the index is
* copied to the remote destination in HDFS.
*
* @param job
* the job to setup.
* @param b
* the boolean to true enable, false to disable.
*/
public static void setIndexLocally(Job job, boolean b) {
setIndexLocally(job.getConfiguration(), b);
}
/**
* Enabled by default, this will enable local indexing on the machine where
* the task is running. Then when the {@link RecordWriter} closes the index is
* copied to the remote destination in HDFS.
*
* @param configuration
* the configuration to setup.
* @param b
* the boolean to true enable, false to disable.
*/
public static void setIndexLocally(Configuration configuration, boolean b) {
configuration.setBoolean(BLUR_OUTPUT_INDEXLOCALLY, b);
}
public static boolean isIndexLocally(Configuration configuration) {
return configuration.getBoolean(BLUR_OUTPUT_INDEXLOCALLY, true);
}
/**
* Enabled by default, this will optimize the index while copying from the
* local index to the remote destination in HDFS. Used in conjunction with the
* setIndexLocally.
*
* @param job
* the job to setup.
* @param b
* the boolean to true enable, false to disable.
*/
public static void setOptimizeInFlight(Job job, boolean b) {
setOptimizeInFlight(job.getConfiguration(), b);
}
/**
* Enabled by default, this will optimize the index while copying from the
* local index to the remote destination in HDFS. Used in conjunction with the
* setIndexLocally.
*
* @param job
* the job to setup.
* @param b
* the boolean to true enable, false to disable.
*/
public static void setOptimizeInFlight(Configuration configuration, boolean b) {
configuration.setBoolean(BLUR_OUTPUT_OPTIMIZEINFLIGHT, b);
}
public static boolean isOptimizeInFlight(Configuration configuration) {
return configuration.getBoolean(BLUR_OUTPUT_OPTIMIZEINFLIGHT, true);
}
/**
* Sets up the output portion of the map reduce job. This does effect the map
* side of the job, of a map and reduce job.
*
* @param job
* the job to setup.
* @param tableDescriptor
* the table descriptor to write the output of the indexing job.
* @throws IOException
*/
public static void setupJob(Job job, TableDescriptor tableDescriptor) throws IOException {
job.setReducerClass(DefaultBlurReducer.class);
job.setNumReduceTasks(tableDescriptor.getShardCount());
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(BlurMutate.class);
job.setOutputFormatClass(BlurOutputFormat.class);
setTableDescriptor(job, tableDescriptor);
BlurMapReduceUtil.addDependencyJars(job);
BlurMapReduceUtil.addAllJarsInBlurLib(job.getConfiguration());
}
}