Package org.apache.flume.source.avro

Examples of org.apache.flume.source.avro.AvroSourceProtocol


        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here


        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client;
    NettyTransceiver nettyTransceiver;
    if (clientEnableCompression) {

      nettyTransceiver = new NettyTransceiver(new InetSocketAddress(
          selectedPort), new CompressionChannelFactory(compressionLevel));

      client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, nettyTransceiver);
    } else {
      nettyTransceiver = new NettyTransceiver(new InetSocketAddress(selectedPort));

      client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, nettyTransceiver);
    }

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here

        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
        selectedPort), new SSLChannelFactory()));

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here

        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client;
    NettyTransceiver nettyTransceiver = null;
    try {
      if (testWithSSL) {
        nettyTransceiver = new NettyTransceiver(
          new InetSocketAddress (dest, selectedPort),
          new SSLChannelFactory());
        client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, nettyTransceiver);
      } else {
        nettyTransceiver = new NettyTransceiver(
          new InetSocketAddress (dest, selectedPort));
        client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, nettyTransceiver);
      }

      AvroFlumeEvent avroEvent = new AvroFlumeEvent();
      avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
      avroEvent.setBody(ByteBuffer.wrap("Hello avro ipFilter".getBytes()));

      logger.info("Client about to append");
      Status status = client.append(avroEvent);
      logger.info("Client appended");
      Assert.assertEquals(Status.OK, status);
    } catch (IOException e) {
      Assert.assertTrue("Should have been allowed: " + ruleDefinition,
        !eventShouldBeAllowed);
View Full Code Here

        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here

        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client;
    if (clientEnableCompression) {
      client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
              selectedPort), new CompressionChannelFactory(6)));
    } else {
      client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
              selectedPort)));
    }

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here

        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
        selectedPort), new SSLChannelFactory()));

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
    avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here

    Transceiver transceiver = null;
    BufferedReader reader = null;

    try {
      transceiver = new NettyTransceiver(new InetSocketAddress(hostname, port));
      AvroSourceProtocol client = SpecificRequestor.getClient(
          AvroSourceProtocol.class, transceiver);
      List<AvroFlumeEvent> eventBuffer = new ArrayList<AvroFlumeEvent>();

      if (fileName != null) {
        reader = new BufferedReader(new FileReader(new File(fileName)));
      } else {
        reader = new BufferedReader(new InputStreamReader(System.in));
      }

      String line = null;
      long lastCheck = System.currentTimeMillis();
      long sentBytes = 0;

      while ((line = reader.readLine()) != null) {
        // logger.debug("read:{}", line);

        if (eventBuffer.size() >= 1000) {
          Status status = client.appendBatch(eventBuffer);

          if (!status.equals(Status.OK)) {
            logger.error("Unable to send batch size:{} status:{}",
                eventBuffer.size(), status);
          }

          eventBuffer.clear();
        }

        AvroFlumeEvent avroEvent = new AvroFlumeEvent();

        avroEvent.headers = new HashMap<CharSequence, CharSequence>();
        avroEvent.body = ByteBuffer.wrap(line.getBytes());

        eventBuffer.add(avroEvent);

        sentBytes += avroEvent.body.capacity();
        sent++;

        long now = System.currentTimeMillis();

        if (now >= lastCheck + 5000) {
          logger.debug("Packed {} bytes, {} events", sentBytes, sent);
          lastCheck = now;
        }
      }

      if (eventBuffer.size() > 0) {
        Status status = client.appendBatch(eventBuffer);

        if (!status.equals(Status.OK)) {
          logger.error("Unable to send batch size:{} status:{}",
              eventBuffer.size(), status);
        }
View Full Code Here

        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    AvroSourceProtocol client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));

    AvroFlumeEvent avroEvent = new AvroFlumeEvent();

    avroEvent.headers = new HashMap<CharSequence, CharSequence>();
    avroEvent.body = ByteBuffer.wrap("Hello avro".getBytes());

    Status status = client.append(avroEvent);

    Assert.assertEquals(Status.OK, status);

    Transaction transaction = channel.getTransaction();
    transaction.begin();
View Full Code Here

            Agent agent = agents[index];
            i = 0;
            do {
                try {
                    transceiver = null;
                    AvroSourceProtocol c = connect(agent.getHost(), agent.getPort());
                    Status status = (batch == null) ? c.append(avroEvent) : c.appendBatch(batch);
                    if (!status.equals(Status.OK)) {
                        if (i == retries - 1) {
                            String warnMsg = "RPC communication failed to " + getName() + " at " +
                                agent.getHost() + ":" + agent.getPort();
                            LOGGER.warn(warnMsg);
View Full Code Here

TOP

Related Classes of org.apache.flume.source.avro.AvroSourceProtocol

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.