Package org.apache.spark.streaming

Examples of org.apache.spark.streaming.Duration

42
43
44
45
46
47
48
49
50
51
52
    }

    StreamingExamples.setStreamingLogLevels();

    // Create the context
    JavaStreamingContext ssc = new JavaStreamingContext(args[0], "QueueStream", new Duration(1000),
            System.getenv("SPARK_HOME"), JavaStreamingContext.jarOfClass(JavaQueueStream.class));

    // Create the queue through which RDDs can be pushed to
    // a QueueInputDStream
    Queue<JavaRDD<Integer>> rddQueue = new LinkedList<JavaRDD<Integer>>();
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    StreamingExamples.setStreamingLogLevels();

    // Create the context with a 1 second batch size
    JavaStreamingContext jssc = new JavaStreamingContext(args[0], "KafkaWordCount",
            new Duration(2000), System.getenv("SPARK_HOME"),
            JavaStreamingContext.jarOfClass(JavaKafkaWordCount.class));

    int numThreads = Integer.parseInt(args[4]);
    Map<String, Integer> topicMap = new HashMap<String, Integer>();
    String[] topics = args[3].split(",");
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    StreamingExamples.setStreamingLogLevels();

    // Create the context with a 1 second batch size
    JavaStreamingContext ssc = new JavaStreamingContext(args[0], "JavaNetworkWordCount",
            new Duration(1000), System.getenv("SPARK_HOME"),
            JavaStreamingContext.jarOfClass(JavaNetworkWordCount.class));

    // Create a NetworkInputDStream on target ip:port and count the
    // words in input stream of \n delimited text (eg. generated by 'nc')
    JavaDStream<String> lines = ssc.socketTextStream(args[1], Integer.parseInt(args[2]));
View Full Code Here
31
32
33
34
35
36
37
38
39
40
public class JavaKinesisStreamSuite extends LocalJavaStreamingContext {
  @Test
  public void testKinesisStream() {
    // Tests the API, does not actually test data receiving
    JavaDStream<byte[]> kinesisStream = KinesisUtils.createStream(ssc, "mySparkStream",
        "https://kinesis.us-west-2.amazonaws.com", new Duration(2000),
        InitialPositionInStream.LATEST, StorageLevel.MEMORY_AND_DISK_2());
   
    ssc.stop();
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        /* Populate the appropriate variables from the given args */
        String streamName = args[0];
        String endpointUrl = args[1];
        /* Set the batch interval to a fixed 2000 millis (2 seconds) */
        Duration batchInterval = new Duration(2000);

        /* Create a Kinesis client in order to determine the number of shards for the given stream */
        AmazonKinesisClient kinesisClient = new AmazonKinesisClient(
                new DefaultAWSCredentialsProviderChain());
        kinesisClient.setEndpoint(endpointUrl);

        /* Determine the number of shards from the stream */
        int numShards = kinesisClient.describeStream(streamName)
                .getStreamDescription().getShards().size();

        /* In this example, we're going to create 1 Kinesis Worker/Receiver/DStream for each shard */
        int numStreams = numShards;

        /* Setup the Spark config. */
        SparkConf sparkConfig = new SparkConf().setAppName("KinesisWordCount");

        /* Kinesis checkpoint interval.  Same as batchInterval for this example. */
        Duration checkpointInterval = batchInterval;

        /* Setup the StreamingContext */
        JavaStreamingContext jssc = new JavaStreamingContext(sparkConfig, batchInterval);

        /* Create the same number of Kinesis DStreams/Receivers as Kinesis stream's shards */
 
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    StreamingExamples.setStreamingLogLevels();

    // Create the context with a 1 second batch size
    SparkConf sparkConf = new SparkConf().setAppName("JavaCustomReceiver");
    JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000));

    // Create a input stream with the custom receiver on target ip:port and count the
    // words in input stream of \n delimited text (eg. generated by 'nc')
    JavaReceiverInputDStream<String> lines = ssc.receiverStream(
      new JavaCustomReceiver(args[0], Integer.parseInt(args[1])));
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    }

    StreamingExamples.setStreamingLogLevels();
    SparkConf sparkConf = new SparkConf().setAppName("JavaKafkaWordCount");
    // Create the context with a 1 second batch size
    JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, new Duration(2000));

    int numThreads = Integer.parseInt(args[3]);
    Map<String, Integer> topicMap = new HashMap<String, Integer>();
    String[] topics = args[2].split(",");
    for (String topic: topics) {
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    StreamingExamples.setStreamingLogLevels();

    // Create the context with a 1 second batch size
    SparkConf sparkConf = new SparkConf().setAppName("JavaNetworkWordCount");
    JavaStreamingContext ssc = new JavaStreamingContext(sparkConf,  new Duration(1000));

    // Create a JavaReceiverInputDStream on target ip:port and count the
    // words in input stream of \n delimited text (eg. generated by 'nc')
    // Note that no duplication in storage level only for running locally.
    // Replication necessary in distributed scenario for fault tolerance.
View Full Code Here
49
50
51
52
53
54
55
  @Override
  public void setUp() {
    testSuite.beforeFunction();
    System.clearProperty("spark.driver.port");
    //System.setProperty("spark.streaming.clock", "org.apache.spark.streaming.util.SystemClock");
    ssc = new JavaStreamingContext("local[2]", "test", new Duration(1000));
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    StreamingExamples.setStreamingLogLevels();
    SparkConf sparkConf = new SparkConf().setAppName("JavaQueueStream");

    // Create the context
    JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000));

    // Create the queue through which RDDs can be pushed to
    // a QueueInputDStream
    Queue<JavaRDD<Integer>> rddQueue = new LinkedList<JavaRDD<Integer>>();
View Full Code Here
TOP

Related Classes of org.apache.spark.streaming.Duration

  • com.cloudera.oryx.lambda.BatchLayer
  • com.cloudera.oryx.lambda.speed.SpeedLayer
  • org.apache.spark.examples.streaming.JavaCustomReceiver
  • org.apache.spark.examples.streaming.JavaKafkaWordCount
  • org.apache.spark.examples.streaming.JavaKinesisWordCountASL
  • org.apache.spark.examples.streaming.JavaNetworkWordCount
  • org.apache.spark.examples.streaming.JavaQueueStream
  • org.apache.spark.streaming.examples.JavaKafkaWordCount
  • org.apache.spark.streaming.examples.JavaNetworkWordCount
  • org.apache.spark.streaming.examples.JavaQueueStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.