document1.put("name", new GenericData.Array(documentSchema.getField("name").schema(), Arrays.asList(name4)));
morphline = createMorphline("test-morphlines/extractAvroPaths");
{
deleteAllDocuments();
Record record = new Record();
record.put(Fields.ATTACHMENT_BODY, document0);
startSession();
// System.out.println(documentSchema.toString(true));
// System.out.println(document0.toString());
assertTrue(morphline.process(record));
assertEquals(1, collector.getRecords().size());
assertEquals(Arrays.asList(10), collector.getFirstRecord().get("/docId"));
assertEquals(Arrays.asList(Arrays.asList()), collector.getFirstRecord().get("/links/backward"));
List expected = Arrays.asList(Arrays.asList(20, 40, 60));
assertEquals(expected, collector.getFirstRecord().get("/links/forward"));
assertEquals(expected, collector.getFirstRecord().get("/links/forward/[]"));
assertEquals(expected, collector.getFirstRecord().get("/links/forward[]"));
assertEquals(Arrays.asList("en-us", "en", "en-gb"), collector.getFirstRecord().get("/name/[]/language/[]/code"));
assertEquals(Arrays.asList("en-us", "en", "en-gb"), collector.getFirstRecord().get("/name[]/language[]/code"));
assertEquals(Arrays.asList("us", "gb"), collector.getFirstRecord().get("/name/[]/language/[]/country"));
assertEquals(Arrays.asList("us", "gb"), collector.getFirstRecord().get("/name[]/language[]/country"));
assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField"));
}
morphline = createMorphline("test-morphlines/extractAvroPathsFlattened");
{
deleteAllDocuments();
Record record = new Record();
record.put(Fields.ATTACHMENT_BODY, document0);
startSession();
// System.out.println(documentSchema.toString(true));
// System.out.println(document0.toString());
assertTrue(morphline.process(record));
assertEquals(1, collector.getRecords().size());
assertEquals(Arrays.asList(10), collector.getFirstRecord().get("/docId"));
assertEquals(Arrays.asList(20, 40, 60), collector.getFirstRecord().get("/links"));
assertEquals(Arrays.asList(), collector.getFirstRecord().get("/links/backward"));
List expected = Arrays.asList(20, 40, 60);
assertEquals(expected, collector.getFirstRecord().get("/links/forward"));
assertEquals(expected, collector.getFirstRecord().get("/links/forward/[]"));
assertEquals(expected, collector.getFirstRecord().get("/links/forward[]"));
assertEquals(Arrays.asList("en-us", "en", "en-gb"), collector.getFirstRecord().get("/name/[]/language/[]/code"));
assertEquals(Arrays.asList("en-us", "en", "en-gb"), collector.getFirstRecord().get("/name[]/language[]/code"));
assertEquals(Arrays.asList("us", "gb"), collector.getFirstRecord().get("/name/[]/language/[]/country"));
assertEquals(Arrays.asList("us", "gb"), collector.getFirstRecord().get("/name[]/language[]/country"));
assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField"));
expected = Arrays.asList("en-us", "us", "en", "http://A", "http://B", "en-gb", "gb");
assertEquals(expected, collector.getFirstRecord().get("/name"));
}
ingestAndVerifyAvro(documentSchema, document0);
ingestAndVerifyAvro(documentSchema, document0, document1);
Record event = new Record();
event.getFields().put(Fields.ATTACHMENT_BODY, document0);
morphline = createMorphline("test-morphlines/extractAvroTree");
deleteAllDocuments();
// System.out.println(document0);
assertTrue(load(event));
assertEquals(1, queryResultSetSize("*:*"));
Record first = collector.getFirstRecord();
assertEquals(Arrays.asList("us", "gb"), first.get("/name/language/country"));
assertEquals(Arrays.asList("en-us", "en", "en-gb"), first.get("/name/language/code"));
assertEquals(Arrays.asList(20, 40, 60), first.get("/links/forward"));
assertEquals(Arrays.asList("http://A", "http://B"), first.get("/name/url"));
assertEquals(Arrays.asList(10), first.get("/docId"));
AbstractParser.removeAttachments(first);
assertEquals(5, first.getFields().asMap().size());
{
morphline = createMorphline("test-morphlines/toAvro");
Record jdoc1 = new Record();
jdoc1.put("_dataset_descriptor_schema", documentSchema);
jdoc1.put("docId", 20);
jdoc1.put("links",
ImmutableMap.of(
"backward", Arrays.asList(10, 30),
"forward", Arrays.asList(80))
);
jdoc1.getFields().putAll("name",
Arrays.asList(
ImmutableMap.of(
"language", Arrays.asList(),
"url", "http://C"))
);
Record expect1 = jdoc1.copy();
expect1.put(Fields.ATTACHMENT_BODY, document1);
processAndVerifySuccess(jdoc1, expect1, false);
Record jdoc0 = new Record();
jdoc0.put("_dataset_descriptor_schema", documentSchema);
jdoc0.put("docId", 10);
jdoc0.put("links",
ImmutableMap.of(
"backward", Arrays.asList(),
"forward", Arrays.asList(20, 40, 60))
);
jdoc0.getFields().putAll("name",
Arrays.asList(
ImmutableMap.of(
"language", new ArrayList(Arrays.asList(
ImmutableMap.of("code", "en-us", "country", "us"),
ImmutableMap.of("code", "en"))),
"url", "http://A"),
ImmutableMap.of(
"language", Arrays.asList(),
"url", "http://B"),
ImmutableMap.of(
"language", new ArrayList(Arrays.asList(
ImmutableMap.of("code", "en-gb", "country", "gb")))
)
)
);
Record expect0 = jdoc0.copy();
expect0.put(Fields.ATTACHMENT_BODY, document0);
processAndVerifySuccess(jdoc0, expect0, false);
}
}