Package com.cloudera.cdk.morphline.base

Examples of com.cloudera.cdk.morphline.base.Compiler

85
86
87
88
89
90
91
92
    String morphlineId = context.getString(MORPHLINE_ID_PARAM);
    if (morphlineFile == null || morphlineFile.trim().length() == 0) {
      throw new MorphlineCompilationException("Missing parameter: " + MORPHLINE_FILE_PARAM, null);
    }
    Config override = ConfigFactory.parseMap(context.getSubProperties(MORPHLINE_VARIABLE_PARAM + "."));
    morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);     
    morphlineFileAndId = morphlineFile + "@" + morphlineId;
  }
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
  }
 
  @Test
  public void testCompile() throws Exception {
    String file = "test-morphlines/pipeWithTwoBasicCommands";
    morphline = new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        "",
        new MorphlineContext.Builder().build(),
        null);
    assertNotNull(morphline);
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
  }
 
  @Test
  public void testCompileWithExplicitMorphlineId() throws Exception {
    String file = "test-morphlines/pipeWithTwoBasicCommands";
    morphline = new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        "morphline1",
        new MorphlineContext.Builder().build(),
        null);
    assertNotNull(morphline);
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
 
  @Test
  public void testCompileWithUnknownMorphlineId() throws Exception {
    String file = "test-morphlines/pipeWithTwoBasicCommands";
    try {
      new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        "morphline2",
        new MorphlineContext.Builder().build(),
        null);
      fail();
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
 
  @Test
  public void testCompileWithMissingMorphline() throws Exception {
    String file = "test-morphlines/compileWithMissingMorphline";
    try {
      new Compiler().compile(
          new File(RESOURCES_DIR + "/" + file + ".conf"),
          "morphline1",
          new MorphlineContext.Builder().build(),
          null);
      fail();
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
  public static void main(String[] args) throws IOException {
    // compile morphline.conf file on the fly
    File morphlineFile = new File(args[0]);
    String morphlineId = null;
    MorphlineContext morphlineContext = new MorphlineContext.Builder().build();
    Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null);
   
    // process each input data file
    Notifications.notifyBeginTransaction(morphline);
    try {
      for (int i = 1; i < args.length; i++) {
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
        ConfigFactory.parseMap(
            ImmutableMap.of("inputFile", inputFile.toString(), "outputDir", cwd.toString())));   
  }
 
  private Command createMorphline(String file, Config... overrides) {
    return new Compiler().compile(
        new File(RESOURCES_DIR + "/" + file + ".conf"),
        null,
        createMorphlineContext(),
        null,
        overrides);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
    morphContext = createMorphlineContext();
    return new PipeBuilder().build(config, null, collector, morphContext);
  }
 
  protected Config parse(String file, Config... overrides) throws IOException {
    Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), overrides);
    config = config.getConfigList("morphlines").get(0);
    Preconditions.checkNotNull(config);
    return config;
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
    SolrLocator locator = new SolrLocator(createMorphlineContext());
    locator.setCollectionName(collection);
    locator.setZkHost(zkServer.getZkAddress());
    //locator.setServerUrl(cloudJettys.get(0).url); // TODO: download IndexSchema from solrUrl not yet implemented
    //locator.setSolrHomeDir(SOLR_HOME_DIR.getPath());
    Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR"));
    config = config.getConfigList("morphlines").get(0);
    return createMorphline(config);
  }
View Full Code Here
207
208
209
210
211
212
213
214
215
  }
 
  private Config parse(String file) throws IOException {
    SolrLocator locator = new SolrLocator(createMorphlineContext());
    locator.setSolrHomeDir(testSolrHome + "/collection1");
    Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR"));
    config = config.getConfigList("morphlines").get(0);
    return config;
  }
View Full Code Here
TOP

Related Classes of com.cloudera.cdk.morphline.base.Compiler

  • com.cloudera.cdk.morphline.api.AbstractMorphlineTest
  • com.cloudera.cdk.morphline.api.Command
  • com.cloudera.cdk.morphline.api.MorphlineDemo
  • com.cloudera.cdk.morphline.api.MorphlineTest
  • com.cloudera.cdk.morphline.hadoop.core.DownloadHdfsFileTest
  • com.cloudera.cdk.morphline.solr.AbstractSolrMorphlineTest
  • com.cloudera.cdk.morphline.solr.AbstractSolrMorphlineZkTest
  • com.cloudera.cdk.morphline.stdlib.DropRecordBuilder
  • com.cloudera.cdk.morphline.stdlib.PipeBuilder
  • com.typesafe.config.Config
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.