Package org.kitesdk.morphline.base

Examples of org.kitesdk.morphline.base.Compiler

96
97
98
99
100
101
102
103
104
105
106
        .setMetricRegistry(SharedMetricRegistries.getOrCreate(morphlineFileAndId))
        .build();
    }
   
    Config override = ConfigFactory.parseMap(context.getSubProperties(MORPHLINE_VARIABLE_PARAM + "."));
    morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);     
   
    this.mappingTimer = morphlineContext.getMetricRegistry().timer(
        MetricRegistry.name("morphline.app", Metrics.ELAPSED_TIME));
    this.numRecords = morphlineContext.getMetricRegistry().meter(
        MetricRegistry.name("morphline.app", Metrics.NUM_RECORDS));
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
            if (entry.getKey().startsWith(variablePrefix)) {
                morphlineVariables.put(entry.getKey().substring(variablePrefix.length()), entry.getValue());
            }
        }
        Config override = ConfigFactory.parseMap(morphlineVariables);
        this.morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, collector,
                override);
       
        for (Map.Entry<String,String> entry : params.entrySet()) {
            String fieldPrefix = MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".";
            if (entry.getKey().startsWith(fieldPrefix)) {
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
  public static void main(String[] args) throws IOException {
    // compile morphline.conf file on the fly
    File morphlineFile = new File(args[0]);
    String morphlineId = null;
    MorphlineContext morphlineContext = new MorphlineContext.Builder().build();
    Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null);
   
    // process each input data file
    Notifications.notifyBeginTransaction(morphline);
    try {
      for (int i = 1; i < args.length; i++) {
View Full Code Here
294
295
296
297
298
299
300
301
302
303
304
  }
 
  @Test
  public void testCompile() throws Exception {
    String file = "test-morphlines/pipeWithTwoBasicCommands";
    morphline = new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        "",
        new MorphlineContext.Builder().build(),
        null);
    assertNotNull(morphline);
View Full Code Here
308
309
310
311
312
313
314
315
316
317
318
  }
 
  @Test
  public void testCompileWithExplicitMorphlineId() throws Exception {
    String file = "test-morphlines/pipeWithTwoBasicCommands";
    morphline = new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        "morphline1",
        new MorphlineContext.Builder().build(),
        null);
    assertNotNull(morphline);
View Full Code Here
320
321
322
323
324
325
326
327
328
329
330
 
  @Test
  public void testCompileWithUnknownMorphlineId() throws Exception {
    String file = "test-morphlines/pipeWithTwoBasicCommands";
    try {
      new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        "morphline2",
        new MorphlineContext.Builder().build(),
        null);
      fail();
View Full Code Here
335
336
337
338
339
340
341
342
343
344
345
 
  @Test
  public void testCompileWithMissingMorphline() throws Exception {
    String file = "test-morphlines/compileWithMissingMorphline";
    try {
      new Compiler().compile(
          new File(RESOURCES_DIR + "/" + file + ".conf"),
          "morphline1",
          new MorphlineContext.Builder().build(),
          null);
      fail();
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    expected.put(Fields.MESSAGE, msg);
    Assert.assertEquals(expected, collector.getFirstRecord());
  }

  private Command createMorphline(String file) {
    return new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        null,
        createMorphlineContext(),
        collector);
  }
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
        ConfigFactory.parseMap(
            ImmutableMap.of("inputFile", inputFile.toString(), "outputDir", cwd.toString())));   
  }
 
  private Command createMorphline(String file, Config... overrides) {
    return new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        null,
        createMorphlineContext(),
        null,
        overrides);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
    morphContext = new MorphlineContext.Builder().setMetricRegistry(new MetricRegistry()).build();
    return new PipeBuilder().build(config, null, collector, morphContext);
  }
 
  protected Config parse(String file, Config... overrides) throws IOException {
    Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), overrides);
    config = config.getConfigList("morphlines").get(0);
    Preconditions.checkNotNull(config);
    return config;
  }
View Full Code Here
TOP

Related Classes of org.kitesdk.morphline.base.Compiler

  • com.ngdata.hbaseindexer.morphline.LocalMorphlineResultToSolrMapper
  • com.typesafe.config.Config
  • org.apache.flume.sink.solr.morphline.MorphlineHandlerImpl
  • org.apache.solr.crunch.MorphlineFn
  • org.apache.solr.hadoop.morphline.MorphlineMapRunner
  • org.kitesdk.morphline.api.AbstractMorphlineTest
  • org.kitesdk.morphline.api.Command
  • org.kitesdk.morphline.api.MorphlineDemo
  • org.kitesdk.morphline.api.MorphlineTest
  • org.kitesdk.morphline.hadoop.core.DownloadHdfsFileTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.