/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.segment.realtime.plumber;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.metamx.common.IAE;
import com.metamx.common.ISE;
import io.druid.data.input.InputRow;
import io.druid.offheap.OffheapBufferPool;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.segment.incremental.IncrementalIndex;
import io.druid.segment.incremental.IncrementalIndexSchema;
import io.druid.segment.incremental.OffheapIncrementalIndex;
import io.druid.segment.indexing.DataSchema;
import io.druid.segment.indexing.RealtimeTuningConfig;
import io.druid.segment.realtime.FireHydrant;
import io.druid.timeline.DataSegment;
import org.joda.time.Interval;
import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
/**
*/
public class Sink implements Iterable<FireHydrant>
{
private final Object hydrantLock = new Object();
private final Interval interval;
private final DataSchema schema;
private final RealtimeTuningConfig config;
private final String version;
private final CopyOnWriteArrayList<FireHydrant> hydrants = new CopyOnWriteArrayList<FireHydrant>();
private volatile FireHydrant currHydrant;
public Sink(
Interval interval,
DataSchema schema,
RealtimeTuningConfig config,
String version
)
{
this.schema = schema;
this.config = config;
this.interval = interval;
this.version = version;
makeNewCurrIndex(interval.getStartMillis(), schema);
}
public Sink(
Interval interval,
DataSchema schema,
RealtimeTuningConfig config,
String version,
List<FireHydrant> hydrants
)
{
this.schema = schema;
this.config = config;
this.interval = interval;
this.version = version;
for (int i = 0; i < hydrants.size(); ++i) {
final FireHydrant hydrant = hydrants.get(i);
if (hydrant.getCount() != i) {
throw new ISE("hydrant[%s] not the right count[%s]", hydrant, i);
}
}
this.hydrants.addAll(hydrants);
makeNewCurrIndex(interval.getStartMillis(), schema);
}
public String getVersion()
{
return version;
}
public Interval getInterval()
{
return interval;
}
public FireHydrant getCurrHydrant()
{
return currHydrant;
}
public int add(InputRow row)
{
if (currHydrant == null) {
throw new IAE("No currHydrant but given row[%s]", row);
}
synchronized (hydrantLock) {
IncrementalIndex index = currHydrant.getIndex();
if (index == null) {
return -1; // the hydrant was swapped without being replaced
}
return index.add(row);
}
}
public boolean isEmpty()
{
synchronized (hydrantLock) {
return hydrants.size() == 1 && currHydrant.getIndex().isEmpty();
}
}
/**
* If currHydrant is A, creates a new index B, sets currHydrant to B and returns A.
*
* @return the current index after swapping in a new one
*/
public FireHydrant swap()
{
return makeNewCurrIndex(interval.getStartMillis(), schema);
}
public boolean swappable()
{
synchronized (hydrantLock) {
return currHydrant.getIndex() != null && currHydrant.getIndex().size() != 0;
}
}
public DataSegment getSegment()
{
return new DataSegment(
schema.getDataSource(),
interval,
version,
ImmutableMap.<String, Object>of(),
Lists.<String>newArrayList(),
Lists.transform(
Arrays.asList(schema.getAggregators()), new Function<AggregatorFactory, String>()
{
@Override
public String apply(@Nullable AggregatorFactory input)
{
return input.getName();
}
}
),
config.getShardSpec(),
null,
0
);
}
private FireHydrant makeNewCurrIndex(long minTimestamp, DataSchema schema)
{
int aggsSize = 0;
for (AggregatorFactory agg : schema.getAggregators()) {
aggsSize += agg.getMaxIntermediateSize();
}
int bufferSize = aggsSize * config.getMaxRowsInMemory();
final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder()
.withMinTimestamp(minTimestamp)
.withQueryGranularity(schema.getGranularitySpec().getQueryGranularity())
.withDimensionsSpec(schema.getParser())
.withMetrics(schema.getAggregators())
.build();
final IncrementalIndex newIndex;
if (config.isIngestOffheap()) {
newIndex = new OffheapIncrementalIndex(
indexSchema,
new OffheapBufferPool(bufferSize)
);
} else {
newIndex = new IncrementalIndex(
indexSchema,
new OffheapBufferPool(bufferSize)
);
}
final FireHydrant old;
synchronized (hydrantLock) {
old = currHydrant;
currHydrant = new FireHydrant(newIndex, hydrants.size(), getSegment().getIdentifier());
hydrants.add(currHydrant);
}
return old;
}
@Override
public Iterator<FireHydrant> iterator()
{
return Iterators.filter(
hydrants.iterator(),
new Predicate<FireHydrant>()
{
@Override
public boolean apply(@Nullable FireHydrant input)
{
final IncrementalIndex index = input.getIndex();
return index == null || index.size() != 0;
}
}
);
}
@Override
public String toString()
{
return "Sink{" +
"interval=" + interval +
", schema=" + schema +
'}';
}
}