Package org.apache.crunch.impl.spark

Examples of org.apache.crunch.impl.spark.SparkPipeline

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    p.done();
  }

  @Test
  public void testUnionTableWithEmptyMR() throws Exception {
    Pipeline p = new SparkPipeline("local", "empty");
    assertFalse(Iterables.isEmpty(p.emptyPTable(Writables.tableOf(Writables.strings(), Writables.longs()))
        .union(
            p.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")))
                .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs())))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }
View Full Code Here
51
52
53
54
55
56
57
58
  private transient Pipeline pipeline;

  @Before
  public void setUp() throws Exception {
    pipeline = new SparkPipeline("local", "tfidf");

  }
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
      SparkConf sconf = new SparkConf();
      if (!sconf.contains("spark.app.name") || sconf.get("spark.app.name").equals(getClass().getName())) {
        sconf.setAppName(Utils.getShortClassName(getClass()));
      }
      JavaSparkContext sparkContext = new JavaSparkContext(sconf);
      pipeline = new SparkPipeline(sparkContext, sparkContext.appName());
      pipeline.setConfiguration(getConf());
    } else {
      throw new IllegalArgumentException("Unsupported --pipeline-type: " + opts.pipelineType);
    }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.spark.SparkPipeline

  • org.apache.crunch.impl.spark.collect.SparkCollectFactory
  • org.apache.crunch.PipelineExecution
  • org.apache.crunch.PipelineResult
  • org.apache.crunch.SkipPTypesIT
  • org.apache.crunch.SparkAggregatorIT
  • org.apache.crunch.SparkCogroupIT
  • org.apache.crunch.SparkEmptyPCollectionIT
  • org.apache.crunch.SparkMapsideJoinIT
  • org.apache.crunch.SparkPageRankIT
  • org.apache.crunch.SparkPipelineCallableIT
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.