Package org.apache.spark

Examples of org.apache.spark.SparkConf

61
62
63
64
65
66
67
68
69
70
71
    if (args.length < 4) {
      System.err.println(
        "Usage: JavaALS <ratings_file> <rank> <iterations> <output_dir> [<blocks>]");
      System.exit(1);
    }
    SparkConf sparkConf = new SparkConf().setAppName("JavaALS");
    int rank = Integer.parseInt(args[1]);
    int iterations = Integer.parseInt(args[2]);
    String outputDir = args[3];
    int blocks = -1;
    if (args.length == 5) {
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  public static void main(String[] args) {
    if (args.length != 3) {
      System.err.println("Usage: JavaLR <input_dir> <step_size> <niters>");
      System.exit(1);
    }
    SparkConf sparkConf = new SparkConf().setAppName("JavaLR");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = sc.textFile(args[0]);
    JavaRDD<LabeledPoint> points = lines.map(new ParsePoint()).cache();
    double stepSize = Double.parseDouble(args[1]);
    int iterations = Integer.parseInt(args[2]);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    int runs = 1;

    if (args.length >= 4) {
      runs = Integer.parseInt(args[3]);
    }
    SparkConf sparkConf = new SparkConf().setAppName("JavaKMeans");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = sc.textFile(inputFile);

    JavaRDD<Vector> points = lines.map(new ParsePoint());
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
      algo = args[1];
    }
    if (args.length > 2) {
      usage();
    }
    SparkConf sparkConf = new SparkConf().setAppName("JavaGradientBoostedTrees");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);

    JavaRDD<LabeledPoint> data = MLUtils.loadLibSVMFile(sc.sc(), datapath).toJavaRDD().cache();

    // Set parameters.
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    log.info("Starting SparkContext for master {}, interval {} seconds",
             streamingMaster, generationIntervalSec);

    long blockIntervalMS = TimeUnit.MILLISECONDS.convert(blockIntervalSec, TimeUnit.SECONDS);

    SparkConf sparkConf = new SparkConf();
    sparkConf.setIfMissing("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
    sparkConf.setIfMissing("spark.streaming.blockInterval", Long.toString(blockIntervalMS));
    sparkConf.setIfMissing("spark.cleaner.ttl", Integer.toString(20 * generationIntervalSec));
    sparkConf.setIfMissing("spark.logConf", "true");
    sparkConf.setMaster(streamingMaster);
    sparkConf.setAppName("OryxSpeedLayer");
    final long batchDurationMS =
        TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS);
    final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf);

    JavaStreamingContextFactory streamingContextFactory = new JavaStreamingContextFactory() {
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    log.info("Starting SparkContext for master {}, interval {} seconds",
             streamingMaster, generationIntervalSec);

    long blockIntervalMS = TimeUnit.MILLISECONDS.convert(blockIntervalSec, TimeUnit.SECONDS);

    SparkConf sparkConf = new SparkConf();
    sparkConf.setIfMissing("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
    sparkConf.setIfMissing("spark.streaming.blockInterval", Long.toString(blockIntervalMS));
    sparkConf.setIfMissing("spark.cleaner.ttl", Integer.toString(20 * generationIntervalSec));
    sparkConf.setIfMissing("spark.logConf", "true");
    sparkConf.setMaster(streamingMaster);
    sparkConf.setAppName("OryxBatchLayer");
    final long batchDurationMS =
       TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS);
    final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf);

    JavaStreamingContextFactory streamingContextFactory = new JavaStreamingContextFactory() {
View Full Code Here
TOP

Related Classes of org.apache.spark.SparkConf

  • co.cask.cdap.internal.app.runtime.spark.AbstractSparkContext
  • com.cloudera.oryx.lambda.BatchLayer
  • com.cloudera.oryx.lambda.speed.SpeedLayer
  • org.apache.crunch.impl.spark.SparkPipeline
  • org.apache.solr.crunch.CrunchIndexerTool
  • org.apache.spark.examples.JavaHdfsLR
  • org.apache.spark.examples.JavaLogQuery
  • org.apache.spark.examples.JavaPageRank
  • org.apache.spark.examples.JavaSparkPi
  • org.apache.spark.examples.JavaStatusAPIDemo
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.