Examples of textFile()

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

370
371
372
373
374
375
376
377
378
379
380
  // Play area

  public static void main(String[] args) {
    final JavaSparkContext sc = new JavaSparkContext("local[1]", "optiq");
    final JavaRDD<String> file = sc.textFile("/usr/share/dict/words");
    System.out.println(
        file.map(
            new Function<String, Object>() {
              @Override
              public Object call(String s) throws Exception {
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

369
370
371
372
373
374
375
376
377
378
379
  // Play area

  public static void main(String[] args) {
    final JavaSparkContext sc = new JavaSparkContext("local[1]", "optiq");
    final JavaRDD<String> file = sc.textFile("/usr/share/dict/words");
    System.out.println(
        file.map(
            new Function<String, Object>() {
              @Override
              public Object call(String s) throws Exception {
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

38
39
40
41
42
43
44
45
46
47
48
      System.exit(1);
    }

    JavaSparkContext ctx = new JavaSparkContext(args[0], "JavaWordCount",
        System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaWordCount.class));
    JavaRDD<String> lines = ctx.textFile(args[1], 1);

    JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
      @Override
      public Iterable<String> call(String s) {
        return Arrays.asList(SPACE.split(s));
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

56
57
58
59
60
61
62
63
64
65
66
      System.exit(1);
    }

    JavaSparkContext sc = new JavaSparkContext(args[0], "JavaLR",
        System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaLR.class));
    JavaRDD<String> lines = sc.textFile(args[1]);
    JavaRDD<LabeledPoint> points = lines.map(new ParsePoint()).cache();
    double stepSize = Double.parseDouble(args[2]);
    int iterations = Integer.parseInt(args[3]);

    // Another way to configure LogisticRegression
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

108
109
110
111
112
113
114
115
116
117
118
      System.exit(1);
    }

    JavaSparkContext sc = new JavaSparkContext(args[0], "JavaHdfsLR",
        System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaHdfsLR.class));
    JavaRDD<String> lines = sc.textFile(args[1]);
    JavaRDD<DataPoint> points = lines.map(new ParsePoint()).cache();
    int ITERATIONS = Integer.parseInt(args[2]);

    // Initialize w to a random value
    double[] w = new double[D];
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

64
65
66
67
68
69
70
71
72
73
74
      runs = Integer.parseInt(args[4]);
    }

    JavaSparkContext sc = new JavaSparkContext(args[0], "JavaKMeans",
        System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaKMeans.class));
    JavaRDD<String> lines = sc.textFile(inputFile);

    JavaRDD<Vector> points = lines.map(new ParsePoint());

    KMeansModel model = KMeans.train(points.rdd(), k, iterations, runs, KMeans.K_MEANS_PARALLEL());
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

104
105
106
107
108
109
110
111
112
113
114
    }

    JavaSparkContext jsc = new JavaSparkContext(args[0], "JavaLogQuery",
      System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaLogQuery.class));

    JavaRDD<String> dataSet = (args.length == 2) ? jsc.textFile(args[1]) : jsc.parallelize(exampleApacheLogs);

    JavaPairRDD<Tuple3<String, String, String>, Stats> extracted = dataSet.mapToPair(new PairFunction<String, Tuple3<String, String, String>, Stats>() {
      @Override
      public Tuple2<Tuple3<String, String, String>, Stats> call(String s) {
        return new Tuple2<Tuple3<String, String, String>, Stats>(extractKey(s), extractStats(s));
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

65
66
67
68
69
70
71
72
73
74
75
    // Loads in input file. It should be in format of:
    //     URL         neighbor URL
    //     URL         neighbor URL
    //     URL         neighbor URL
    //     ...
    JavaRDD<String> lines = ctx.textFile(args[1], 1);

    // Loads all URLs from input file and initialize their neighbors.
    JavaPairRDD<String, Iterable<String>> links = lines.mapToPair(new PairFunction<String, String, String>() {
      @Override
      public Tuple2<String, String> call(String s) {
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

71
72
73
74
75
76
77
78
79
80
81
      blocks = Integer.parseInt(args[5]);
    }

    JavaSparkContext sc = new JavaSparkContext(args[0], "JavaALS",
        System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaALS.class));
    JavaRDD<String> lines = sc.textFile(args[1]);

    JavaRDD<Rating> ratings = lines.map(new ParseRating());

    MatrixFactorizationModel model = ALS.train(ratings.rdd(), rank, iterations, 0.01, blocks);
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

55
56
57
58
59
60
61
62
63
64
65
    JavaSparkContext ctx = new JavaSparkContext("local", "JavaSparkSQL",
        System.getenv("SPARK_HOME"), JavaSparkContext.jarOfClass(JavaSparkSQL.class));
    JavaSQLContext sqlCtx = new JavaSQLContext(ctx);

    // Load a text file and convert each line to a Java Bean.
    JavaRDD<Person> people = ctx.textFile("examples/src/main/resources/people.txt").map(
      new Function<String, Person>() {
        public Person call(String line) throws Exception {
          String[] parts = line.split(",");

          Person person = new Person();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.