Examples of com.cloudera.cdk.morphline.api.Command

com.cloudera.cdk.morphline.api.Command
A command transforms a record into zero or more records. A command has a boolean return code, indicating success or failure. All record handlers in a morphline implement this interface. Commands are chained together. The parent of a command A is the command B that passes records to A. A is the child of B. Data is sent on the data plane whereas notifications are sent on the control plane, which is a separate communication channel.

    return detect(event, includeMetaData, false);
  }
  
  private String detect(Record event, boolean includeMetaData, boolean excludeParameters) throws IOException {
    List key = Arrays.asList(includeMetaData, excludeParameters);
    Command cachedMorphline = morphlineCache.get(key);
    if (cachedMorphline == null) { // avoid recompiling time and again (performance)
      Config override = ConfigFactory.parseString("INCLUDE_META_DATA : " + includeMetaData + "\nEXCLUDE_PARAMETERS : " + excludeParameters);
      cachedMorphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypesAndFile", override);
      morphlineCache.put(key, cachedMorphline);
    }
    collector.reset();
    assertTrue(cachedMorphline.process(event));
    String mimeType = (String) collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE);
    return mimeType;
  }

View Full Code Here

    }
    Assert.assertFalse(dst.exists());
    new File(cwd, fileName).mkdirs(); // will be auto deleted!
    Files.write("wrong msg", new File(new File(cwd, fileName), fileName), Charsets.UTF_8); // will be auto deleted!


    Command morphline = createMorphline("test-morphlines/testDownloadHdfsFile", inputFile, cwd);         
    Assert.assertTrue(morphline.process(new Record()));    
    Assert.assertEquals(msg, Files.toString(dst, Charsets.UTF_8));
    if (isDir) {
      FileUtil.fullyDelete(dst.getParentFile());
    } else {
      FileUtil.fullyDelete(dst);
    }
    Assert.assertTrue(fileSystem.exists(inputFile));
    Assert.assertTrue(FileUtil.fullyDelete(cwd));
    
    // verify that subsequent calls with same inputFile won't copy the file again (to prevent races)
    morphline = createMorphline("test-morphlines/downloadHdfsFile", inputFile, cwd);       
    Assert.assertTrue(morphline.process(new Record()));    
    Assert.assertFalse(dst.exists());
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertFalse(dst.exists());
    Assert.assertFalse(cwd.exists());
    
    Assert.assertTrue(fileSystem.delete(inputFile, true));

View Full Code Here

            try {
              int iters = 0;
              MorphlineContext ctx = new MorphlineContext.Builder().build();
              Config config = parse("test-morphlines/convertHTML");
              Collector myCollector = new Collector();
              Command myMorphline = new PipeBuilder().build(config, null, myCollector, ctx);
              
              long start = System.currentTimeMillis();
              while (System.currentTimeMillis() < start + durationMillis) {
                Record record = new Record();
                record.put("id", "123");

View Full Code Here

    private Command elseChain;
    
    public IfThenElse(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
      super(builder, config, parent, child, context);
      
      Command devNull = new DropRecordBuilder().build(null, this, null, context); // pipes into /dev/null
      List<Command> conditions = buildCommandChain(config, "conditions", devNull, true);
      if (conditions.size() == 0) {
        throw new MorphlineCompilationException("Missing conditions", config);
      } else {
        this.conditionChain = conditions.get(0);

View Full Code Here

   *          avoid sending a notification multiple times to finalChild, once from each branch.
   */
  protected List<Command> buildCommandChain(Config rootConfig, String configKey, Command finalChild, boolean ignoreNotifications) {    
    List<? extends Config> commandConfigs = new Configs().getConfigList(rootConfig, configKey, Collections.EMPTY_LIST);
    List<Command> commands = new ArrayList();
    Command currentParent = this;
    Connector lastConnector = null;        
    for (int i = 0; i < commandConfigs.size(); i++) {
      boolean isLast = (i == commandConfigs.size() - 1);
      Connector connector = new Connector(ignoreNotifications && isLast);
      if (isLast) {
        connector.setChild(finalChild);
      }
      Config cmdConfig = commandConfigs.get(i);
      Command cmd = buildCommand(cmdConfig, currentParent, connector);
      commands.add(cmd);
      if (i > 0) {
        lastConnector.setChild(cmd);
      }
      connector.setParent(cmd);

View Full Code Here

    if (!(obj instanceof CommandBuilder)) {
      throw new MorphlineCompilationException("Type of command " + cmdName + " must be an instance of "
          + CommandBuilder.class.getName() + " but is: " + cmdClass.getName(), cmdConfig);
    } 
    CommandBuilder builder = (CommandBuilder) obj;
    Command cmd = builder.build(cmdConfig.getConfig(cmdName), currentParent, finalChild, getContext());
    return cmd;
  }

View Full Code Here

      config = parse(morphlineFile, overrides);
    } catch (IOException e) {
      throw new MorphlineCompilationException("Cannot parse morphline file: " + morphlineFile, null, e);
    }
    Config morphlineConfig = find(morphlineId, config, morphlineFile.getPath());
    Command morphlineCommand = compile(morphlineConfig, morphlineContext, finalChild);
    return morphlineCommand;
  }

View Full Code Here

TOP

Related Classes of com.cloudera.cdk.morphline.api.Command

com.cloudera.cdk.morphline.base.AbstractCommand

com.cloudera.cdk.morphline.base.Compiler

com.cloudera.cdk.morphline.hadoop.core.DownloadHdfsFileTest

com.cloudera.cdk.morphline.saxon.TagsoupMorphlineTest

com.cloudera.cdk.morphline.stdlib.IfThenElseBuilder$IfThenElse

com.cloudera.cdk.morphline.tika.DetectMimeTypesTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.