Package com.cloudera.cdk.morphline.api

Examples of com.cloudera.cdk.morphline.api.Record

309
310
311
312
313
314
315
316
317
    assertFalse(iter.hasNext());
  }
 
  private ListMultimap<String, Object> next(Iterator<SolrDocument> iter) {
    SolrDocument doc = iter.next();
    Record record = toRecord(doc);
    record.removeAll("_version_"); // the values of this field are unknown and internal to solr
    return record.getFields();   
  }
View Full Code Here
315
316
317
318
319
320
321
322
323
324
325
    record.removeAll("_version_"); // the values of this field are unknown and internal to solr
    return record.getFields();   
  }
 
  private Record toRecord(SolrDocument doc) {
    Record record = new Record();
    for (String key : doc.keySet()) {
      record.getFields().replaceValues(key, doc.getFieldValues(key));       
    }
    return record;
  }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    morphlineFileAndId = morphlineFile + "@" + morphlineId;
  }

  @Override
  public void process(Event event) {
    Record record = new Record();
    for (Entry<String, String> entry : event.getHeaders().entrySet()) {
      record.put(entry.getKey(), entry.getValue());
    }
    byte[] bytes = event.getBody();
    if (bytes != null && bytes.length > 0) {
      record.put(Fields.ATTACHMENT_BODY, bytes);
    }   
    try {
      Notifications.notifyStartSession(morphline);
      if (!morphline.process(record)) {
        LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
  }

  private void testBasic(String morphlineFile) throws Exception {
    morphline = createMorphline(morphlineFile);   
     
    Record record = new Record();
    String userAgentStr = "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3";
    record.put("user_agents", userAgentStr);
    Record expected = new Record();
    expected.put("user_agents", userAgentStr);
    expected.put("ua_family", "Mobile Safari");
    expected.put("device_family", "iPhone");
    expected.put("string_constant1", "foo");
    expected.put("string_constant2", "");
    expected.put("ua_family_and_version", "Mobile Safari/5.1");
    expected.put("os_family_and_version", "iOS5.1.1-foo@bar");
           
    processAndVerifySuccess(record, expected, true);
   
    record = new Record();
    record.put("user_agents", userAgentStr);
    processAndVerifySuccess(record, expected, true);
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
      this.valueField = getConfigs().getString(config, ReadSequenceFileBuilder.CONFIG_VALUE_FIELD, Fields.ATTACHMENT_BODY);
    }

    @Override
    protected boolean doProcess(Record inputRecord) {
      Record outputRecord = inputRecord.copy();

      // change key
      Text myTextKey = (Text)inputRecord.getFirstValue(this.keyField);
      outputRecord.replaceValues(this.keyField, MyWritable.keyStr(myTextKey));
      // change value
      MyWritable myWritableValue = (MyWritable)inputRecord.getFirstValue(this.valueField);
      outputRecord.replaceValues(this.valueField, MyWritable.valueStr(myWritableValue));
      return super.doProcess(outputRecord);
    }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
    String path = RESOURCES_DIR;
    File sequenceFile = new File(path, "testSequenceFileContentSimple.seq");
    int numRecords = 5;
    HashMap<String, Record>  expected = createTextSequenceFile(sequenceFile, numRecords);
    InputStream in = new FileInputStream(sequenceFile.getAbsolutePath());
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    startSession();

    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));
    assertTrue(areFieldsEqual(expected, collector.getRecords()));
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
        SequenceFile.CompressionType.NONE, null, metadata);
      for (int i = 0; i < numRecords; ++i) {
        Text key = new Text("key" + i);
        Text value = new Text("value" + i);
        writer.append(key, value);
        Record record = new Record();
        record.put("key", key);
        record.put("value", value);
        map.put(key.toString(), record);
      }
    } finally {
      Closeables.closeQuietly(writer);
    }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
    String path = RESOURCES_DIR;
    File sequenceFile = new File(path, "testSequenceFileContentCustomParsers.seq");
    int numRecords = 10;
    HashMap<String, Record> expected = createTextSequenceFile(sequenceFile, numRecords);
    InputStream in = new FileInputStream(sequenceFile.getAbsolutePath());
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    startSession();

    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));
   
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
        SequenceFile.CompressionType.NONE, null, metadata);
      for (int i = 0; i < numRecords; ++i) {
        Text key = new Text("key" + i);
        ParseTextMyWritableBuilder.MyWritable value = new ParseTextMyWritableBuilder.MyWritable("value", i);
        writer.append(key, value);
        Record record = new Record();
        record.put("key", key);
        record.put("value", value);
        map.put(key.toString(), record);
      }
    } finally {
      Closeables.closeQuietly(writer);
    }
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
    if (expected.size() != actual.size()) {
      return false;
    }
    for (Record current : actual) {
      String key = current.getFirstValue("key").toString();
      Record currentExpected = expected.get(key);
      if (!areRecordFieldsEqual(current, currentExpected, Arrays.asList(new String[]{"key", "value"}))) {
        return false;
      }
    }
View Full Code Here
TOP

Related Classes of com.cloudera.cdk.morphline.api.Record

  • com.cloudera.cdk.morphline.avro.AvroMorphlineTest
  • com.cloudera.cdk.morphline.avro.ExtractAvroPathsBuilder$ExtractAvroPaths
  • com.cloudera.cdk.morphline.avro.ExtractAvroTreeBuilder$ExtractAvroTree
  • com.cloudera.cdk.morphline.avro.ReadAvroBuilder$ReadAvro
  • com.cloudera.cdk.morphline.avro.ReadAvroContainerBuilder$ReadAvroContainer
  • com.cloudera.cdk.morphline.avro.ToAvroBuilder$ToAvro
  • com.cloudera.cdk.morphline.avro.WriteAvroToByteArrayBuilder$WriteAvroToByteArray
  • com.cloudera.cdk.morphline.base.Notifications
  • com.cloudera.cdk.morphline.hadoop.core.DownloadHdfsFileTest
  • com.cloudera.cdk.morphline.hadoop.rcfile.ReadRCFileBuilder$ReadRCFile
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.