Package org.kitesdk.morphline.api

Examples of org.kitesdk.morphline.api.Record


      validateArguments();
    }

    @Override
    protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException {
      Record template = inputRecord.copy();
      removeAttachments(template);
      template.removeAll(Fields.MESSAGE);
      Charset detectedCharset = detectCharset(inputRecord, charset)
      Reader reader = new InputStreamReader(stream, detectedCharset);
      BufferedReader lineReader = new BufferedReader(reader, getBufferSize(stream));
      StringBuilder lines = null;
      String line;
     
      while ((line = lineReader.readLine()) != null) {
        if (lines == null) {
          lines = new StringBuilder(line);
        } else {
          boolean isMatch = regex.reset(line).matches();
          if (negate) {
            isMatch = !isMatch;
          }
          /*
          not match && previous --> do next
          not match && next     --> do previous
          match && previous     --> do previous
          match && next         --> do next            
          */
          boolean doPrevious = (what == What.previous);
          if (!isMatch) {
            doPrevious = !doPrevious;
          }
         
          if (doPrevious) { // do previous
            lines.append('\n');
            lines.append(line);
          } else {          // do next
            if (lines.length() > 0 && !flushRecord(template.copy(), lines.toString())) {
              return false;
            }
            lines.setLength(0);
            lines.append(line);             
          }
        }         
      }
      if (lines != null && lines.length() > 0) {
        return flushRecord(template.copy(), lines.toString());
      }
      return true;
    }
View Full Code Here


        }
      }
     
      SolrInputDocument doc = handler.newDocument();
      LOG.debug("solr doc: {}", doc);     
      Record outputRecord = toRecord(doc);
      return getChild().process(outputRecord);
    }
View Full Code Here

          + factoryClass.getName(), config, e);
      }
    }

    private Record toRecord(SolrInputDocument doc) {
      Record record = new Record();
      for (Entry<String, SolrInputField> entry : doc.entrySet()) {
        record.getFields().putAll(entry.getKey(), entry.getValue().getValues());       
      }
      return record;
    }
View Full Code Here

  @Test
  public void testXQueryTweetTexts() throws Exception {
    morphline = createMorphline("test-morphlines/xquery-tweet-texts");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.xml"));
    Record record = new Record();
    record.put("id", "123");
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record,
        ImmutableMultimap.of("id", "123", "text", "sample tweet one"),
        ImmutableMultimap.of("id", "123", "text", "sample tweet two")
        );   
    in.close();
View Full Code Here

  @Test
  public void testXQueryTweetUsers() throws Exception {
    morphline = createMorphline("test-morphlines/xquery-tweet-users");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.xml"));
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record,
        ImmutableMultimap.of("followers_count", "111", "id", "11111112", "screen_name", "fake_user1", "greeting", "hello world", "annotation", "An XSLT Morphline"),
        ImmutableMultimap.of("followers_count", "222", "id", "222223", "screen_name", "fake_user2", "greeting", "hello world", "annotation", "An XSLT Morphline")
        );   
    in.close();
View Full Code Here

  @Test
  public void testXQueryAtomFeeds() throws Exception {
    morphline = createMorphline("test-morphlines/xquery-atom-feeds");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/atom.xml"));
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record,
        ImmutableMultimap
            .of("id",
                "tag:blogger.com,1999:blog-10832468.post-112136653221060965",
                "summary",
View Full Code Here

  @Test
  public void testXQueryShakespeareSpeakers() throws Exception {
    morphline = createMorphline("test-morphlines/xquery-shakespeare-speakers");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/othello.xml"));
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record,
        ImmutableMultimap.of("name", "OTHELLO", "frequency", "274"),
        ImmutableMultimap.of("name", "IAGO", "frequency", "272"),
        ImmutableMultimap.of("name", "DESDEMONA", "frequency", "165")
        );   
View Full Code Here

  @Test
  public void testXQueryAtomicValues() throws Exception {
    morphline = createMorphline("test-morphlines/xquery-atomic-values");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.xml"));
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record);
    in.close();
 
View Full Code Here

  @Test
  public void testXsltIdentityHelloWorld() throws Exception {
    morphline = createMorphline("test-morphlines/xslt-helloworld-identity");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/helloworld.xml"));
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record,
        ImmutableMultimap.of("description", "An XSLT Morphline", "welcome", "Hello, World!", "id", "2")
        );   
    in.close();
 
View Full Code Here

  @Test
  public void testXsltHelloWorldSequence() throws Exception {
    morphline = createMorphline("test-morphlines/xslt-helloworld-sequence");   
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/helloworld.xml"));
    Record record = new Record();
    record.put("id", "123");
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record,
        ImmutableMultimap.of("id", "123", "attr", "foo", "HEAD", "title1", "BODY", "Hello, World!Paragraph1aParagraph1b"),
        ImmutableMultimap.of("id", "123", "HEAD", "title2", "BODY", "Hello, World!Paragraph2aParagraph2b")
        );   
    in.close();
View Full Code Here

TOP

Related Classes of org.kitesdk.morphline.api.Record

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.