Examples of org.kitesdk.morphline.api.Record

org.kitesdk.morphline.api.Record
A record is a set of named fields where each field has a list of one or more values. A value can be of any type, i.e. any Java Object. That is, a record is a {@link ListMultimap} asin Guava’s {@link ArrayListMultimap}. Note that a field can be multi-valued and that any two records need not use common field names. This flexible data model corresponds exactly to the characteristics of the Solr/Lucene data model (i.e. a record is a SolrInputDocument). A field with zero values is removed from the record - it does not exist as such.

    assertFalse(iter.hasNext());
  }
  
  private ListMultimap<String, Object> next(Iterator<SolrDocument> iter) {
    SolrDocument doc = iter.next();
    Record record = toRecord(doc);
    record.removeAll("_version_"); // the values of this field are unknown and internal to solr
    return record.getFields();    
  }

View Full Code Here

    record.removeAll("_version_"); // the values of this field are unknown and internal to solr
    return record.getFields();    
  }
  
  private Record toRecord(SolrDocument doc) {
    Record record = new Record();
    for (String key : doc.keySet()) {
      record.getFields().replaceValues(key, doc.getFieldValues(key));        
    }
    return record;
  }

View Full Code Here

  @Override
  public void process(Event event) {
    numRecords.mark();
    Timer.Context timerContext = mappingTimer.time();
    try {
      Record record = new Record();
      for (Entry<String, String> entry : event.getHeaders().entrySet()) {
        record.put(entry.getKey(), entry.getValue());
      }
      byte[] bytes = event.getBody();
      if (bytes != null && bytes.length > 0) {
        record.put(Fields.ATTACHMENT_BODY, bytes);
      }    
      try {
        Notifications.notifyStartSession(morphline);
        if (!morphline.process(record)) {
          numFailedRecords.mark();

View Full Code Here

    @Override
    public void map(Result result, SolrUpdateWriter solrUpdateWriter) {
        numRecords.mark();
        Timer.Context timerContext = mappingTimer.time();
        try {
            Record record = new Record();
            record.put(Fields.ATTACHMENT_BODY, result);
            record.put(Fields.ATTACHMENT_MIME_TYPE, MorphlineResultToSolrMapper.OUTPUT_MIME_TYPE);
            for (Map.Entry<String, String> entry : forcedRecordFields.entrySet()) {
                record.replaceValues(entry.getKey(), entry.getValue());
            }
            collector.reset(solrUpdateWriter);
            try {
                Notifications.notifyStartSession(morphline);
                if (!morphline.process(record)) {

View Full Code Here

        
        assertFalse(resultMapper.containsRequiredData(result));
    }


    private Record toRecord(SolrInputDocument doc) {
      Record record = new Record();
      for (Entry<String, SolrInputField> entry : doc.entrySet()) {
          record.getFields().putAll(entry.getKey(), entry.getValue().getValues());
      }
      return record;
    }

View Full Code Here

   * Notify a command that a lifecycle event has occurred.
   * @param command The {@link Command} to be notified.
   * @param event The {@link LifecycleEvent} to be passed down to the given command.
   */
  private static void notify(Command command, LifecycleEvent event) {
    Record notification = new Record();
    notification.put(LIFE_CYCLE, event);
    command.notify(notification);
  }

View Full Code Here

      validateArguments();
    }
  
    @Override
    protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException {
      Record template = inputRecord.copy();
      removeAttachments(template);
      template.removeAll(Fields.MESSAGE);
      Charset detectedCharset = detectCharset(inputRecord, charset);  
      Reader reader = new InputStreamReader(stream, detectedCharset);
      BufferedReader lineReader = new BufferedReader(reader, getBufferSize(stream));
      boolean isFirst = true;
      String line;


      while ((line = lineReader.readLine()) != null) {
        if (isFirst && ignoreFirstLine) {
          isFirst = false;
          continue; // ignore first line
        }
        if (line.length() == 0) {
          continue; // ignore empty lines
        }
        if (commentPrefix != null && line.startsWith(commentPrefix)) {
          continue; // ignore comments
        }
        Record outputRecord = template.copy();
        outputRecord.put(Fields.MESSAGE, line);
        incrementNumRecords();
        
        // pass record to next command in chain:
        if (!getChild().process(outputRecord)) {
          return false;

View Full Code Here

      clob.setLength(0);
      int len;
      while ((len = reader.read(buffer)) >= 0) {
        clob.append(buffer, 0, len);
      }
      Record outputRecord = inputRecord.copy();
      removeAttachments(outputRecord);
      outputRecord.replaceValues(outputFieldName, clob.toString());
        
      // pass record to next command in chain:
      return getChild().process(outputRecord);
    }

View Full Code Here

      validateArguments();
    }
  
    @Override
    protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException {
      Record template = inputRecord.copy();
      removeAttachments(template);
      Charset detectedCharset = detectCharset(inputRecord, charset);  
      BufferedReader reader = new BufferedReader(
          new InputStreamReader(stream, detectedCharset), getBufferSize(stream));
      if (ignoreFirstLine) {
        reader.readLine();
      }      


      while (true) {
        Record outputRecord = readNext(reader, template);
        if (outputRecord == null) {
          break;
        }
        incrementNumRecords();

View Full Code Here


        if (commentPrefix.length() > 0 && line.startsWith(commentPrefix)) {
          continue; // ignore
        }


        Record outputRecord = template.copy();
        if (!tokenizer.tokenizeLine(line, reader, outputRecord)) {
          continue; // ignore
        }
        
        return outputRecord;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.kitesdk.morphline.api.Record

com.ngdata.hbaseindexer.morphline.LocalMorphlineResultToSolrMapper

com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapperTest

org.apache.flume.sink.solr.morphline.MorphlineHandlerImpl

org.apache.flume.sink.solr.morphline.TestMorphlineSolrSink

org.apache.solr.crunch.MorphlineFn

org.apache.solr.hadoop.morphline.MorphlineMapRunner

org.kitesdk.examples.morphlines.ExampleMorphlineTest

org.kitesdk.morphline.avro.AvroMorphlineTest

org.kitesdk.morphline.avro.ExtractAvroPathsBuilder$ExtractAvroPaths

org.kitesdk.morphline.avro.ExtractAvroTreeBuilder$ExtractAvroTree

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.