Examples of org.lilyproject.tools.import_.json.IgnoreEmptyFieldsRecordReader

Package org.lilyproject.tools.import_.json

Examples of org.lilyproject.tools.import_.json.IgnoreEmptyFieldsRecordReader

org.lilyproject.tools.import_.json.WriteOptions
A specialized RecordReader which will drop fields containing empty strings, drop lists containing zero entries, and drop nested records containing zero fields. For fields in the root record, the fields are not just skipped, they are actively deleted by adding them to the deleted fields (this last part is actually a feature of the superclass, see {@link RecordReader#deleteNullFields()}).
This makes a lot of sense when the input jsons are generated by ETL jobs that don't want to bother with having to change the structure of the JSON according to whether a field has a value or not. Instead, they can use a fixed json structure into which they just need to fill in the blanks.

        // whole of the file, but use the tree API to load individual items (field types, records, ...).
        // This way things should still work fast and within little memory if anyone would use this to
        // load large amounts of records.


        try {
            namespaces = new NamespacesImpl();


            JsonParser jp = JsonFormat.JSON_FACTORY_NON_STD.createJsonParser(is);


            JsonToken current;
            current = jp.nextToken();

View Full Code Here

        assertEquals(new Integer(55), record1.getField(new QName("import2", "f2")));
    }


    private byte[] scanToBytes(RecordScan scan) throws RepositoryException, InterruptedException, IOException {
        return JsonFormat.serializeAsBytes(
                writer.toJson(scan, new WriteOptions(), new NamespacesImpl(false), repository));
    }

View Full Code Here

                writer.toJson(scan, new WriteOptions(), new NamespacesImpl(false), repository));
    }


    private RecordScan scanFromBytes(byte[] data) throws IOException, RepositoryException, JsonFormatException,
            InterruptedException {
        return reader.fromJson(JsonFormat.deserializeNonStd(data), new NamespacesImpl(false), repository);
    }

View Full Code Here

    }


    @Before
    public void setup() {
        writer = new RecordScanWriter();
        reader = new RecordScanReader();
    }

View Full Code Here

        Closer.close(repoSetup);
    }


    @Before
    public void setup() {
        writer = new RecordScanWriter();
        reader = new RecordScanReader();
    }

View Full Code Here

    }


    public void readNamespaces(ObjectNode node) throws JsonFormatException {
        // We don't expect the namespaces to be modified since we're reading rather than writing, still wrap it
        // to make sure they are really not modified.
        this.namespaces = new UnmodifiableNamespaces(NamespacesConverter.fromJson(node));
    }

View Full Code Here

        if (filter.getField() != null && filter.getFieldValue() != null) {
            node.put("field", QNameConverter.toJson(filter.getField(), namespaces));


            ValueType valueType = repository.getTypeManager().getFieldTypeByName(filter.getField()).getValueType();
            JsonNode valueAsJson = RecordWriter.INSTANCE.valueToJson(filter.getFieldValue(), valueType,
                    new WriteOptions(), namespaces, repository);


            node.put("fieldValue", valueAsJson);
            node.putObject("value");
        }

View Full Code Here

        }
        return integers;
    }


    public static WriteOptions getWriteOptions(UriInfo uriInfo) {
        WriteOptions options = new WriteOptions();


        String includeSchema = uriInfo.getQueryParameters().getFirst("schema");
        if (includeSchema != null) {
            options.setIncludeSchema(includeSchema.equalsIgnoreCase("true"));
        }


        String useNamespacePrefixes = uriInfo.getQueryParameters().getFirst("nsprefixes");
        if (useNamespacePrefixes != null) {
            options.setUseNamespacePrefixes(useNamespacePrefixes.equalsIgnoreCase("true"));
        }


        return options;
    }

View Full Code Here

            job.getConfiguration().set(REPOSITORY_TABLES, Joiner.on(',').join(repositoryTables));
        }


        if (scan != null) {
            try {
                JsonNode node = RecordScanWriter.INSTANCE.toJson(scan, new WriteOptions(), repository);
                String scanData = JsonFormat.serializeAsString(node);
                job.getConfiguration().set(AbstractLilyScanInputFormat.SCAN, scanData);
            } catch (Exception e) {
                ExceptionUtil.handleInterrupt(e);
                throw new RuntimeException(e);

View Full Code Here

        assertEquals(new Integer(55), record1.getField(new QName("import2", "f2")));
    }


    private byte[] scanToBytes(RecordScan scan) throws RepositoryException, InterruptedException, IOException {
        return JsonFormat.serializeAsBytes(
                writer.toJson(scan, new WriteOptions(), new NamespacesImpl(false), repository));
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.lilyproject.tools.import_.json.IgnoreEmptyFieldsRecordReader

com.google.common.base.Splitter

com.ngdata.lily.security.hbase.client.AuthorizationContext

org.codehaus.jackson.JsonNode

org.codehaus.jackson.JsonParser

org.codehaus.jackson.JsonToken

org.codehaus.jackson.map.ObjectMapper

org.codehaus.jackson.node.ArrayNode

org.codehaus.jackson.node.JsonNodeFactory

org.codehaus.jackson.node.ObjectNode

org.joda.time.DateTime

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.