Examples of org.apache.flume.api.RpcClient

org.apache.flume.api.RpcClient

Public client interface for sending data to Flume.

This interface is intended not to change incompatibly for Flume 1.x.

Note: It is recommended for applications to construct {@link RpcClient} instances using the {@link RpcClientFactory} class,instead of using builders associated with a particular implementation class.
@see org.apache.flume.api.RpcClientFactory

  private void run() throws IOException, FlumeException,
      EventDeliveryException {


    LineReader reader = null;


    RpcClient rpcClient = RpcClientFactory.getDefaultInstance(hostname, port,
        BATCH_SIZE);
    try {
      if (fileName != null) {
        reader = new BufferedLineReader(new FileReader(new File(fileName)));
      } else if (dirName != null) {
        reader = new SpoolingFileLineReader(new File(dirName), ".COMPLETED",
            BATCH_SIZE, MAX_LINE_LENGTH);
      }
      else {
        reader = new BufferedLineReader(new InputStreamReader(System.in));
      }




      long lastCheck = System.currentTimeMillis();
      long sentBytes = 0;


      int batchSize = rpcClient.getBatchSize();
      List<String> lines = Lists.newLinkedList();
      while (!(lines = reader.readLines(batchSize)).isEmpty()) {
        List<Event> eventBuffer = Lists.newArrayList();
        for (String line : lines) {
          Event event = EventBuilder.withBody(line, Charsets.UTF_8);
          setHeaders(event);
          eventBuffer.add(event);
          sentBytes += event.getBody().length;
          sent++;


          long now = System.currentTimeMillis();
          if (now >= lastCheck + 5000) {
            logger.debug("Packed {} bytes, {} events", sentBytes, sent);
            lastCheck = now;
          }
        }
        rpcClient.appendBatch(eventBuffer);
      }
      logger.debug("Finished");
    } finally {
      if (reader != null) {
        logger.debug("Closing reader");
        reader.close();
      }


      logger.debug("Closing RPC client");
      rpcClient.close();
    }
  }

View Full Code Here

    context.put("reset-connection-interval", String.valueOf("5"));


    sink.setChannel(channel);
    Configurables.configure(sink, context);
    sink.start();
    RpcClient firstClient = sink.getUnderlyingClient();
    Thread.sleep(6000);
    Transaction t = channel.getTransaction();
    t.begin();
    channel.put(EventBuilder.withBody("This is a test", Charset.defaultCharset()));
    t.commit();

View Full Code Here

  private void run() throws IOException, FlumeException,
      EventDeliveryException {


    BufferedReader reader = null;


    RpcClient rpcClient = RpcClientFactory.getDefaultInstance(hostname, port, BATCH_SIZE);
    try {
      List<Event> eventBuffer = Lists.newArrayList();


      if (fileName != null) {
        reader = new BufferedReader(new FileReader(new File(fileName)));
      } else {
        reader = new BufferedReader(new InputStreamReader(System.in));
      }


      String line;
      long lastCheck = System.currentTimeMillis();
      long sentBytes = 0;


      int batchSize = rpcClient.getBatchSize();
      while ((line = reader.readLine()) != null) {
        // logger.debug("read:{}", line);


        int size = eventBuffer.size();
        if (size == batchSize) {
          rpcClient.appendBatch(eventBuffer);
          eventBuffer.clear();
        }


        Event event = EventBuilder.withBody(line, Charset.forName("UTF8"));
        setHeaders(event);
        eventBuffer.add(event);


        sentBytes += event.getBody().length;
        sent++;


        long now = System.currentTimeMillis();


        if (now >= lastCheck + 5000) {
          logger.debug("Packed {} bytes, {} events", sentBytes, sent);
          lastCheck = now;
        }
      }


      if (!eventBuffer.isEmpty()) {
        rpcClient.appendBatch(eventBuffer);
      }


      logger.debug("Finished");
    } finally {
      if (reader != null) {
        logger.debug("Closing reader");
        reader.close();
      }


      logger.debug("Closing RPC client");
      rpcClient.close();
    }
  }

View Full Code Here

     try {


       StagedInstall.getInstance().startAgent(
         "rpccagent", CONFIG_FILE_PRCCLIENT_TEST);
       StagedInstall.waitUntilPortOpens("localhost", 12121, 20000);
       RpcClient client = RpcClientFactory.getDefaultInstance(
           "localhost", 12121);
       String[] text = {"foo", "bar", "xyz", "abc"};
       for (String str : text) {
         client.append(EventBuilder.withBody(str.getBytes()));
       }


       // Stop the agent
       StagedInstall.getInstance().stopAgent();


       // Try sending the event which should fail
       try {
         client.append(EventBuilder.withBody("test".getBytes()));
         Assert.fail("EventDeliveryException expected but not raised");
       } catch (EventDeliveryException ex) {
         System.out.println("Attempting to close client");
         client.close();
       }
     } finally {
       if (StagedInstall.getInstance().isRunning()) {
         StagedInstall.getInstance().stopAgent();
       }

View Full Code Here

  }


  @Test
  public void testRpcClient() throws Exception {
    StagedInstall.waitUntilPortOpens("localhost", 12121, 20000);
    RpcClient client = RpcClientFactory.getDefaultInstance("localhost", 12121);
    String[] text = {"foo", "bar", "xyz", "abc"};
    for (String str : text) {
      client.append(EventBuilder.withBody(str.getBytes()));
    }
  }

View Full Code Here

    context.put("reset-connection-interval", String.valueOf("5"));


    sink.setChannel(channel);
    Configurables.configure(sink, context);
    sink.start();
    RpcClient firstClient = sink.getUnderlyingClient();
    Thread.sleep(6000);
    // Make sure they are not the same object, connection should be reset
    Assert.assertFalse(firstClient == sink.getUnderlyingClient());
    sink.stop();

View Full Code Here

  private void run() throws IOException, FlumeException,
      EventDeliveryException {


    BufferedReader reader = null;


    RpcClient rpcClient = RpcClientFactory.getInstance(hostname, port, BATCH_SIZE);
    try {
      List<Event> eventBuffer = Lists.newArrayList();


      if (fileName != null) {
        reader = new BufferedReader(new FileReader(new File(fileName)));
      } else {
        reader = new BufferedReader(new InputStreamReader(System.in));
      }


      String line;
      long lastCheck = System.currentTimeMillis();
      long sentBytes = 0;


      int batchSize = rpcClient.getBatchSize();
      while ((line = reader.readLine()) != null) {
        // logger.debug("read:{}", line);


        int size = eventBuffer.size();
        if (size == batchSize) {
          rpcClient.appendBatch(eventBuffer);
          eventBuffer.clear();
        }


        Event event = EventBuilder.withBody(line, Charset.forName("UTF8"));
        eventBuffer.add(event);


        sentBytes += event.getBody().length;
        sent++;


        long now = System.currentTimeMillis();


        if (now >= lastCheck + 5000) {
          logger.debug("Packed {} bytes, {} events", sentBytes, sent);
          lastCheck = now;
        }
      }


      if (!eventBuffer.isEmpty()) {
        rpcClient.appendBatch(eventBuffer);
      }


      logger.debug("Finished");
    } finally {
      if (reader != null) {
        logger.debug("Closing reader");
        reader.close();
      }


      logger.debug("Closing RPC client");
      rpcClient.close();
    }
  }

View Full Code Here

  private void run() throws IOException, FlumeException,
      EventDeliveryException {


    EventReader reader = null;


    RpcClient rpcClient;
    if (rpcClientPropsFile != null) {
      rpcClient = RpcClientFactory.getInstance(new File(rpcClientPropsFile));
    } else {
      rpcClient = RpcClientFactory.getDefaultInstance(hostname, port,
          BATCH_SIZE);
    }


    try {
      if (fileName != null) {
        reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
      } else if (dirName != null) {
        reader = new ReliableSpoolingFileEventReader.Builder()
            .spoolDirectory(new File(dirName)).build();
      } else {
        reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
      }


      long lastCheck = System.currentTimeMillis();
      long sentBytes = 0;


      int batchSize = rpcClient.getBatchSize();
      List<Event> events;
      while (!(events = reader.readEvents(batchSize)).isEmpty()) {
        for (Event event : events) {
          event.setHeaders(headers);
          sentBytes += event.getBody().length;
          sent++;


          long now = System.currentTimeMillis();
          if (now >= lastCheck + 5000) {
            logger.debug("Packed {} bytes, {} events", sentBytes, sent);
            lastCheck = now;
          }
        }
        rpcClient.appendBatch(events);
        if (reader instanceof ReliableEventReader) {
          ((ReliableEventReader) reader).commit();
        }
      }
      logger.debug("Finished");
    } finally {
      if (reader != null) {
        logger.debug("Closing reader");
        reader.close();
      }


      logger.debug("Closing RPC client");
      rpcClient.close();
    }
  }

View Full Code Here

    if (apikey != null) {
      event.getHeaders().put(Constants.Gateway.API_KEY, apikey);
    }


    // event is now fully constructed, ready to send
    RpcClient client = RpcClientFactory.getDefaultInstance(hostname, port, 1);
    try {
      client.append(event);
    } catch (EventDeliveryException e) {
      client.close();
      System.err.println("Error sending flume event: " + e.getMessage());
      return null;
    }
    client.close();
    return "OK.";
  }

View Full Code Here

TOP

Related Classes of org.apache.flume.api.RpcClient

co.cask.cdap.gateway.tools.FlumeClient

org.apache.flume.client.avro.AvroCLIClient

org.apache.flume.sink.TestAvroSink

org.apache.flume.test.agent.TestRpcClient

org.apache.flume.test.agent.TestRpcClientCommunicationFailure

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.