/*
* Copyright (c) 2010 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific
* language governing permissions and limitations under the
* License. See accompanying LICENSE file.
*/
package io.s4.util;
import io.s4.collector.EventWrapper;
import io.s4.dispatcher.partitioner.CompoundKeyInfo;
import io.s4.emitter.CommLayerEmitter;
import io.s4.emitter.EventEmitter;
import io.s4.schema.Schema;
import io.s4.schema.Schema.Property;
import io.s4.serialize.KryoSerDeser;
import io.s4.serialize.SerializerDeserializer;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Array;
import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
public class LoadGenerator {
public static void main(String args[]) {
Options options = new Options();
boolean warmUp = false;
options.addOption(OptionBuilder.withArgName("rate")
.hasArg()
.withDescription("Rate (events per second)")
.create("r"));
options.addOption(OptionBuilder.withArgName("display_rate")
.hasArg()
.withDescription("Display Rate at specified second boundary")
.create("d"));
options.addOption(OptionBuilder.withArgName("start_boundary")
.hasArg()
.withDescription("Start boundary in seconds")
.create("b"));
options.addOption(OptionBuilder.withArgName("run_for")
.hasArg()
.withDescription("Run for a specified number of seconds")
.create("x"));
options.addOption(OptionBuilder.withArgName("cluster_manager")
.hasArg()
.withDescription("Cluster manager")
.create("z"));
options.addOption(OptionBuilder.withArgName("sender_application_name")
.hasArg()
.withDescription("Sender application name")
.create("a"));
options.addOption(OptionBuilder.withArgName("listener_application_name")
.hasArg()
.withDescription("Listener application name")
.create("g"));
options.addOption(OptionBuilder.withArgName("sleep_overhead")
.hasArg()
.withDescription("Sleep overhead")
.create("o"));
options.addOption(new Option("w", "Warm-up"));
CommandLineParser parser = new GnuParser();
CommandLine line = null;
try {
// parse the command line arguments
line = parser.parse(options, args);
} catch (ParseException exp) {
// oops, something went wrong
System.err.println("Parsing failed. Reason: " + exp.getMessage());
System.exit(1);
}
int expectedRate = 250;
if (line.hasOption("r")) {
try {
expectedRate = Integer.parseInt(line.getOptionValue("r"));
} catch (Exception e) {
System.err.println("Bad expected rate specified "
+ line.getOptionValue("r"));
System.exit(1);
}
}
int displayRateIntervalSeconds = 20;
if (line.hasOption("d")) {
try {
displayRateIntervalSeconds = Integer.parseInt(line.getOptionValue("d"));
} catch (Exception e) {
System.err.println("Bad display rate value specified "
+ line.getOptionValue("d"));
System.exit(1);
}
}
int startBoundary = 2;
if (line.hasOption("b")) {
try {
startBoundary = Integer.parseInt(line.getOptionValue("b"));
} catch (Exception e) {
System.err.println("Bad start boundary value specified "
+ line.getOptionValue("b"));
System.exit(1);
}
}
int updateFrequency = 0;
if (line.hasOption("f")) {
try {
updateFrequency = Integer.parseInt(line.getOptionValue("f"));
} catch (Exception e) {
System.err.println("Bad query udpdate frequency specified "
+ line.getOptionValue("f"));
System.exit(1);
}
System.out.printf("Update frequency is %d\n", updateFrequency);
}
int runForTime = 0;
if (line.hasOption("x")) {
try {
runForTime = Integer.parseInt(line.getOptionValue("x"));
} catch (Exception e) {
System.err.println("Bad run for time specified "
+ line.getOptionValue("x"));
System.exit(1);
}
System.out.printf("Run for time is %d\n", runForTime);
}
String clusterManagerAddress = null;
if (line.hasOption("z")) {
clusterManagerAddress = line.getOptionValue("z");
}
String senderApplicationName = null;
if (line.hasOption("a")) {
senderApplicationName = line.getOptionValue("a");
}
String listenerApplicationName = null;
if (line.hasOption("a")) {
listenerApplicationName = line.getOptionValue("g");
}
if (listenerApplicationName == null) {
listenerApplicationName = senderApplicationName;
}
long sleepOverheadMicros = -1;
if (line.hasOption("o")) {
try {
sleepOverheadMicros = Long.parseLong(line.getOptionValue("o"));
} catch (NumberFormatException e) {
System.err.println("Bad sleep overhead specified "
+ line.getOptionValue("o"));
System.exit(1);
}
System.out.printf("Specified sleep overhead is %d\n",
sleepOverheadMicros);
}
if (line.hasOption("w")) {
warmUp = true;
}
List loArgs = line.getArgList();
if (loArgs.size() < 1) {
System.err.println("No input file specified");
System.exit(1);
}
String inputFilename = (String) loArgs.get(0);
EventEmitter emitter = null;
SerializerDeserializer serDeser = new KryoSerDeser();
CommLayerEmitter clEmitter = new CommLayerEmitter();
clEmitter.setAppName(senderApplicationName);
clEmitter.setListenerAppName(listenerApplicationName);
clEmitter.setClusterManagerAddress(clusterManagerAddress);
clEmitter.setSenderId(String.valueOf(System.currentTimeMillis() / 1000));
clEmitter.setSerDeser(serDeser);
clEmitter.init();
emitter = clEmitter;
long endTime = 0;
if (runForTime > 0) {
endTime = System.currentTimeMillis() + (runForTime * 1000);
}
LoadGenerator loadGenerator = new LoadGenerator();
loadGenerator.setInputFilename(inputFilename);
loadGenerator.setEventEmitter(clEmitter);
loadGenerator.setDisplayRateInterval(displayRateIntervalSeconds);
loadGenerator.setExpectedRate(expectedRate);
loadGenerator.run();
System.exit(0);
}
private EventEmitter eventEmitter;
private String inputFilename;
private int emitCount;
private int displayRateInterval = 0;
private int expectedRate = 200;
private int adjustedExpectedRate = 1;
private long sleepOverheadMicros = -1;
private static int PROCESS_TIME_LIST_MAX_SIZE = 15;
private long[] processTimes = new long[PROCESS_TIME_LIST_MAX_SIZE];
private int processTimePointer = 0;
private Map<Integer, EventTypeInfo> eventTypeInfoMap = new HashMap<Integer, EventTypeInfo>();
public int getEmitCount() {
return emitCount;
}
public void setEventEmitter(EventEmitter eventEmitter) {
this.eventEmitter = eventEmitter;
}
public void setInputFilename(String inputFilename) {
this.inputFilename = inputFilename;
}
public void setDisplayRateInterval(int displayRateInterval) {
this.displayRateInterval = displayRateInterval;
}
public void setSleepOverheadMicros(long sleepOverheadMicros) {
this.sleepOverheadMicros = sleepOverheadMicros;
}
public void setExpectedRate(int expectedRate) {
this.expectedRate = expectedRate;
}
private Random rand = new Random(System.currentTimeMillis());
public LoadGenerator() {
if (sleepOverheadMicros == -1) {
// calculate sleep overhead
long totalSleepOverhead = 0;
for (int i = 0; i < 50; i++) {
long startTime = System.nanoTime();
try {
Thread.sleep(1);
} catch (InterruptedException ie) {
}
totalSleepOverhead += (System.nanoTime() - startTime)
- (1 * 1000 * 1000);
}
sleepOverheadMicros = (totalSleepOverhead / 50) / 1000;
}
System.out.println("Sleep overhead is " + sleepOverheadMicros);
}
public void run() {
// for now, no warm-up mechanism
adjustedExpectedRate = expectedRate;
long startTime = 0;
long intervalStart = 0;
int emitCountStart = 0;
long[] rateInfo = new long[2];
rateInfo[0] = 100; // start with a sleep time of 100
BufferedReader br = null;
Reader inputReader = null;
try {
if (inputFilename.equals("-")) {
inputReader = new InputStreamReader(System.in);
} else {
inputReader = new FileReader(inputFilename);
}
br = new BufferedReader(inputReader);
String inputLine = null;
boolean firstLine = true;
EventWrapper eventWrapper = null;
for (startTime = System.nanoTime(); (inputLine = br.readLine()) != null; startTime = System.nanoTime()) {
if (firstLine) {
JSONObject jsonRecord = new JSONObject(inputLine);
createEventTypeInfo(jsonRecord);
System.out.println(eventTypeInfoMap);
if (eventTypeInfoMap.size() == 0) {
return;
}
firstLine = false;
continue;
}
try {
JSONObject jsonRecord = new JSONObject(inputLine);
int classIndex = jsonRecord.getInt("_index");
EventTypeInfo eventTypeInfo = eventTypeInfoMap.get(classIndex);
if (eventTypeInfo == null) {
System.err.printf("Invalid _index value %d\n",
classIndex);
return;
}
Object event = makeRecord(jsonRecord,
eventTypeInfo.getSchema());
eventWrapper = new EventWrapper(eventTypeInfo.getStreamName(),
event,
new ArrayList<CompoundKeyInfo>());
// System.out.println(eventWrapper.getStreamName() + ": " +
// eventWrapper.getEvent());
} catch (Exception e) {
e.printStackTrace();
System.err.printf("Bad input data %s\n", inputLine);
continue;
}
int partition = Math.abs(rand.nextInt())
% eventEmitter.getNodeCount();
eventEmitter.emit(partition, eventWrapper);
emitCount++;
// the rest of the stuff in this block is just to maintain the
// rate
processTimes[processTimePointer] = System.nanoTime()
- startTime;
processTimePointer = (processTimePointer == PROCESS_TIME_LIST_MAX_SIZE - 1) ? 0
: processTimePointer + 1;
if (emitCount == 1 || emitCount % 20 == 0) {
rateInfo = getRateInfo(rateInfo);
}
// if it's time, display the actual emit rate
if (intervalStart == 0) {
intervalStart = System.currentTimeMillis();
} else {
long interval = System.currentTimeMillis() - intervalStart;
if (interval >= (displayRateInterval * 1000)) {
double rate = (emitCount - emitCountStart)
/ (interval / 1000.0);
System.out.println("Rate is " + rate);
intervalStart = System.currentTimeMillis();
emitCountStart = emitCount;
}
}
if (rateInfo[1] == 0 || emitCount % rateInfo[1] == 0) {
try {
Thread.sleep(rateInfo[0]);
} catch (InterruptedException ie) {
}
}
}
System.out.printf("Emitted %d events\n", emitCount);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
try {
br.close();
} catch (Exception e) {
}
try {
inputReader.close();
} catch (Exception e) {
}
}
}
@SuppressWarnings("unchecked")
public void createEventTypeInfo(JSONObject classInfo) {
String className = "";
try {
for (Iterator it = classInfo.keys(); it.hasNext();) {
className = (String) it.next();
JSONObject jsonEventTypeInfo = classInfo.getJSONObject(className);
int classIndex = (Integer) jsonEventTypeInfo.getInt("classIndex");
String streamName = jsonEventTypeInfo.getString("streamName");
Class clazz = Class.forName(className);
Schema schema = new Schema(clazz);
eventTypeInfoMap.put(classIndex, new EventTypeInfo(schema,
streamName));
}
} catch (JSONException je) {
je.printStackTrace();
} catch (ClassNotFoundException cnfe) {
System.err.println("Count not locate class " + className);
}
}
@SuppressWarnings("unchecked")
private Object makeRecord(JSONObject jsonRecord, Schema schema) {
Object event = null;
try {
event = schema.getType().newInstance();
for (Iterator it = jsonRecord.keys(); it.hasNext();) {
String propertyName = (String) it.next();
Property property = schema.getProperties().get(propertyName);
if (property == null) {
continue; // not in schema, just continue
}
Method setterMethod = property.getSetterMethod();
Object value = jsonRecord.get(propertyName);
if (value.equals(JSONObject.NULL)) {
continue;
}
setterMethod.invoke(event, makeSettableValue(property, value));
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return event;
}
@SuppressWarnings("unchecked")
private Object makeSettableValue(Property property, Object value) {
String propertyName = property.getName();
Class propertyType = property.getType();
if (propertyType.isArray()) {
if (!(value instanceof JSONArray)) {
System.err.println("Type mismatch for field " + propertyName);
return null;
}
System.out.println("Is array!");
return makeArray(property, (JSONArray) value);
} else if (property.isList()) {
if (!(value instanceof JSONArray)) {
System.err.println("Type mismatch for field " + propertyName);
return null;
}
return makeList(property, (JSONArray) value);
} else if (propertyType.isPrimitive()) {
if (!(value instanceof Number || value instanceof Boolean)) {
System.err.println("Type mismatch for field " + propertyName
+ "; expected number or boolean, found "
+ value.getClass());
return null;
}
return value; // hmm... does this work?
} else if (propertyType.equals(String.class)) {
if (!(value instanceof String)) {
System.err.println("Type mismatch for field " + propertyName
+ "; expected String, found " + value.getClass());
return null;
}
return value;
} else if (property.isNumber()) {
if (!(value instanceof Integer || value instanceof Long
|| value instanceof Float || value instanceof Double
|| value instanceof BigDecimal || value instanceof BigInteger)) {
return null;
}
Number adjustedValue = (Number) value;
if (propertyType.equals(Long.class) && !(value instanceof Long)) {
adjustedValue = new Long(((Number) value).longValue());
} else if (propertyType.equals(Integer.class)
&& !(value instanceof Integer)) {
adjustedValue = new Integer(((Number) value).intValue());
} else if (propertyType.equals(Double.class)
&& !(value instanceof Double)) {
adjustedValue = new Double(((Number) value).doubleValue());
} else if (propertyType.equals(Float.class)
&& !(value instanceof Float)) {
adjustedValue = new Float(((Number) value).floatValue());
} else if (propertyType.equals(BigDecimal.class)) {
adjustedValue = new BigDecimal(((Number) value).longValue());
} else if (propertyType.equals(BigInteger.class)) {
adjustedValue = BigInteger.valueOf(((Number) value).longValue());
}
return adjustedValue;
} else if (value instanceof JSONObject) {
return makeRecord((JSONObject) value, property.getSchema());
}
return null;
}
public Object makeList(Property property, JSONArray jsonArray) {
Property componentProperty = property.getComponentProperty();
int size = jsonArray.length();
List<Object> list = new ArrayList<Object>(size);
try {
for (int i = 0; i < size; i++) {
Object value = jsonArray.get(i);
list.add(makeSettableValue(componentProperty, value));
}
} catch (JSONException je) {
throw new RuntimeException(je);
}
return list;
}
@SuppressWarnings("unchecked")
public Object makeArray(Property property, JSONArray jsonArray) {
Property componentProperty = property.getComponentProperty();
Class clazz = componentProperty.getType();
int size = jsonArray.length();
Object array = Array.newInstance(clazz, size);
try {
for (int i = 0; i < size; i++) {
Object value = jsonArray.get(i);
Object adjustedValue = makeSettableValue(componentProperty,
value);
Array.set(array, i, adjustedValue);
}
} catch (JSONException je) {
throw new RuntimeException(je);
}
return array;
}
private long[] getRateInfo(long[] rateInfo) {
long totalTimeNanos = 0;
int entryCount = 0;
for (int i = 0; i < processTimes.length; i++) {
if (processTimes[i] == Long.MIN_VALUE) {
break;
}
entryCount++;
totalTimeNanos += processTimes[i];
}
long averageTimeMicros = (long) ((totalTimeNanos / (double) entryCount) / 1000.0);
// fudge the time for additional overhead
averageTimeMicros += (long) (averageTimeMicros * 0.30);
if (emitCount % 5000 == 0) {
// System.out.println("Average time in micros is " +
// averageTimeMicros);
}
long sleepTimeMicros = 0;
long millis = 0;
long timeToMeetRateMicros = adjustedExpectedRate * averageTimeMicros;
long leftOver = 1000000 - timeToMeetRateMicros;
if (leftOver <= 0) {
sleepTimeMicros = 0;
} else {
sleepTimeMicros = (leftOver / adjustedExpectedRate)
- sleepOverheadMicros;
}
// how many events can be processed in the nanos time?
int eventsBeforeSleep = 1;
if (sleepTimeMicros < 1000) {
sleepTimeMicros = 1000 + sleepOverheadMicros;
millis = 1;
double numNapsDouble = ((double) leftOver / sleepTimeMicros);
int numNaps = (int) Math.ceil(numNapsDouble);
if (numNaps > 0) {
eventsBeforeSleep = adjustedExpectedRate / numNaps;
}
if (leftOver <= 0) {
millis = 0;
eventsBeforeSleep = 1000;
}
} else {
millis = sleepTimeMicros / 1000;
}
rateInfo[0] = millis;
rateInfo[1] = eventsBeforeSleep;
return rateInfo;
}
static class EventTypeInfo {
private Schema schema;
private String streamName;
public EventTypeInfo(Schema schema, String streamName) {
this.schema = schema;
this.streamName = streamName;
}
public Schema getSchema() {
return schema;
}
public String getStreamName() {
return streamName;
}
}
}