Examples of com.cloudera.cdk.morphline.api.Record

com.cloudera.cdk.morphline.api.Record
A record is a set of named fields where each field has a list of one or more values. A value can be of any type, i.e. any Java Object. That is, a record is a {@link ListMultimap} asin Guava’s {@link ArrayListMultimap}. Note that a field can be multi-valued and that any two records need not use common field names. This flexible data model corresponds exactly to the characteristics of the Solr/Lucene data model (i.e. a record is a SolrInputDocument). A field with zero values is removed from the record - it does not exist as such.

  @Test
  public void testReadJsonWithMap() throws Exception {
    morphline = createMorphline("test-morphlines/readJsonWithMap");    
    for (int j = 0; j < 3; j++) { // also test reuse of objects and low level avro buffers
      InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/stream.json"));
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, in);
      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));

View Full Code Here

  @Test
  public void testExtractJsonPaths() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPaths");    
    File file = new File(RESOURCES_DIR + "/test-documents/arrays.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));

View Full Code Here

  @Test
  public void testExtractJsonPathsFlattened() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPathsFlattened");    
    File file = new File(RESOURCES_DIR + "/test-documents/arrays.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));

View Full Code Here

    
    @Override
    protected boolean doProcess(Record inputRecord) {
      JsonNode datum = (JsonNode) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY);
      Preconditions.checkNotNull(datum);
      Record outputRecord = inputRecord.copy();
      
      for (Map.Entry<String, Collection<String>> entry : stepMap.entrySet()) {
        String fieldName = entry.getKey();
        List<String> steps = (List<String>) entry.getValue();
        extractPath(datum, fieldName, steps, outputRecord, 0);

View Full Code Here

  @Test
  public void testComplexDocuments() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPaths");    
    File file = new File(RESOURCES_DIR + "/test-documents/complex.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));

View Full Code Here

  @Test
  public void testExtractJsonPathsComplexFlattened() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPathsFlattened");    
    File file = new File(RESOURCES_DIR + "/test-documents/complex.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));        
    assertEquals(1, collector.getRecords().size());

View Full Code Here

    byte[] bytes = Files.toByteArray(file);
    long start = System.currentTimeMillis();
    long duration = durationSecs * 1000;
    int iters = 0; 
    while (System.currentTimeMillis() < start + duration) {
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, bytes);      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));    
      iters++;

View Full Code Here

        while (iter.hasNextValue()) {
          Object rootNode = iter.nextValue();
          incrementNumRecords();
          LOG.debug("jsonObject: {}", rootNode);
          
          Record outputRecord = inputRecord.copy();
          removeAttachments(outputRecord);
          outputRecord.put(Fields.ATTACHMENT_BODY, rootNode);
          outputRecord.put(Fields.ATTACHMENT_MIME_TYPE, MIME_TYPE);
  
          // pass record to next command in chain:
          if (!getChild().process(outputRecord)) {
            return false;
          }

View Full Code Here

    morphline = createMorphline("test-morphlines/rcFileMorphlineRow");
    String rcFileName = "testRCFileRowWise.rc";
    List<Record> expected = setupRCFile(rcFileName, NUM_RECORDS, NUM_COLUMNS,
        true);
    Path inputFile = dfs.makeQualified(new Path(testDirectory, rcFileName));
    Record input = new Record();
    input.put(Fields.ATTACHMENT_NAME, inputFile.toString());
    input.put(Fields.ATTACHMENT_BODY, readPath(inputFile));
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(input));
    assertTrue(areFieldsEqual(expected, collector.getRecords(), NUM_COLUMNS,
        NUM_RECORDS, true));

View Full Code Here

    morphline = createMorphline("test-morphlines/rcFileMorphlineColumn");
    String rcFileName = "testRCFileColumnWise.rc";
    List<Record> expected = setupRCFile(rcFileName, NUM_RECORDS, NUM_COLUMNS,
        false);
    Path inputFile = dfs.makeQualified(new Path(testDirectory, rcFileName));
    Record input = new Record();
    input.put(Fields.ATTACHMENT_NAME, inputFile.toString());
    input.put(Fields.ATTACHMENT_BODY, readPath(inputFile));
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(input));
    assertTrue(areFieldsEqual(expected, collector.getRecords(), NUM_COLUMNS,
        NUM_RECORDS, false));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.cdk.morphline.api.Record

com.cloudera.cdk.morphline.avro.AvroMorphlineTest

com.cloudera.cdk.morphline.avro.ExtractAvroPathsBuilder$ExtractAvroPaths

com.cloudera.cdk.morphline.avro.ExtractAvroTreeBuilder$ExtractAvroTree

com.cloudera.cdk.morphline.avro.ReadAvroBuilder$ReadAvro

com.cloudera.cdk.morphline.avro.ReadAvroContainerBuilder$ReadAvroContainer

com.cloudera.cdk.morphline.avro.ToAvroBuilder$ToAvro

com.cloudera.cdk.morphline.avro.WriteAvroToByteArrayBuilder$WriteAvroToByteArray

com.cloudera.cdk.morphline.base.Notifications

com.cloudera.cdk.morphline.hadoop.core.DownloadHdfsFileTest

com.cloudera.cdk.morphline.hadoop.rcfile.ReadRCFileBuilder$ReadRCFile

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.