Package com.cloudera.oryx.kafka.util

Examples of com.cloudera.oryx.kafka.util.ProduceData


      int howManyUpdate) throws IOException, InterruptedException {

    int zkPort = getZKPort();
    int kafkaPort = getKafkaBrokerPort();

    updateProducer = new ProduceData(updateGenerator,
        zkPort,
        kafkaPort,
        UPDATE_TOPIC,
        howManyUpdate,
        10);
View Full Code Here


      int intervalMsec) throws IOException, InterruptedException {

    int zkPort = getZKPort();

    int bufferMS = WAIT_BUFFER_IN_WRITES * intervalMsec;
    ProduceData produce = new ProduceData(datumGenerator,
                                          zkPort,
                                          getKafkaBrokerPort(),
                                          INPUT_TOPIC,
                                          howMany,
                                          intervalMsec);

    final List<Pair<String,String>> keyMessages = new ArrayList<>();

    Thread.sleep(bufferMS);

    try (CloseableIterator<Pair<String,String>> data =
             new ConsumeData(UPDATE_TOPIC, zkPort).iterator();
         BatchLayer<?,?,?> batchLayer = new BatchLayer<>(config)) {

      log.info("Starting consumer thread");
      new Thread(new LoggingRunnable() {
        @Override
        public void doRun() {
          while (data.hasNext()) {
            keyMessages.add(data.next());
          }
        }
      }).start();

      log.info("Starting batch layer");
      batchLayer.start();

      // Sleep for a while after starting server to let it init
      Thread.sleep(bufferMS);

      log.info("Producing data");
      produce.start();

      // Sleep for a while before shutting down server to let it finish
      Thread.sleep(bufferMS);

    } finally {
      produce.deleteTopic();
    }

    return keyMessages;
  }
View Full Code Here

    int zkPort = getZKPort();
    int kafkaPort = getKafkaBrokerPort();

    int bufferMS = WAIT_BUFFER_IN_WRITES * 10;

    ProduceData inputProducer = new ProduceData(inputGenerator,
                                                zkPort,
                                                kafkaPort,
                                                INPUT_TOPIC,
                                                howManyInput,
                                                10);
    ProduceData updateProducer = new ProduceData(updateGenerator,
                                                 zkPort,
                                                 kafkaPort,
                                                 UPDATE_TOPIC,
                                                 howManyUpdate,
                                                 10);

    final List<Pair<String,String>> keyMessages = new ArrayList<>();

    Thread.sleep(bufferMS);

    try (CloseableIterator<Pair<String,String>> data =
             new ConsumeData(UPDATE_TOPIC, zkPort).iterator();
         SpeedLayer<?,?,?> speedLayer = new SpeedLayer<>(config)) {

      log.info("Starting consumer thread");
      new Thread(new LoggingRunnable() {
        @Override
        public void doRun() {
          while (data.hasNext()) {
            keyMessages.add(data.next());
          }
        }
      }).start();

      log.info("Starting speed layer");
      speedLayer.start();

      Thread.sleep(bufferMS);

      // Load all updates first
      log.info("Producing updates");
      updateProducer.start();

      // Sleep for a while after starting server to let it init
      Thread.sleep(bufferMS);

      log.info("Producing input");
      inputProducer.start();

      // Sleep for a while before shutting down server to let it finish
      Thread.sleep(bufferMS);

    } finally {
      inputProducer.deleteTopic();
      updateProducer.deleteTopic();
    }

    return keyMessages;
  }
View Full Code Here

TOP

Related Classes of com.cloudera.oryx.kafka.util.ProduceData

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.