Package com.socrata.api

Examples of com.socrata.api.SodaImporter


    protected Soda2Producer createProducer() throws IOException {
        return new Soda2Producer(connect());
    }

    protected SodaImporter createImporter() throws IOException {
        return new SodaImporter(connect());
    }
View Full Code Here


    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, TEST_NOMINATIONA_CSV, "Name");
        try {
            //Get the ID column
            int columnId = ((Dataset)datasetCreated).getRowIdentifierColumnId();
            sodaImporter.removeColumn(datasetCreated.getId(), columnId);


            try {
                sodaImporter.append(datasetCreated.getId(), TEST_NOMINATIONA_CSV, 1, null);
                TestCase.fail("Expected failure after updating a non-existing column as a rowidentifier");
            } catch (MetadataUpdateError mue) {
                 //Success error expected
            }

        } finally {
            sodaImporter.deleteDataset(datasetCreated.getId());
        }

    }
View Full Code Here

    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, WIDE_CSV, "col0");
        try {
            //Get the ID column
            int columnId = ((Dataset)datasetCreated).getRowIdentifierColumnId();
            sodaImporter.removeColumn(datasetCreated.getId(), columnId);

            /*
            try {
                sodaImporter.append(datasetCreated.getId(), WIDE_CSV, 1, null);
                TestCase.fail("Expected failure after updating a non-existing column as a rowidentifier");
            } catch (MetadataUpdateError mue) {
                //Success error expected
            }
            */
        } finally {
            sodaImporter.deleteDataset(datasetCreated.getId());
        }

    }
View Full Code Here

    @Test
    public void testLongUpsert() throws IOException, SodaError, InterruptedException
    {

        final Soda2Producer producer = createProducer();
        final SodaImporter importer = createImporter();

        final String name = "LongUpsertName" + UUID.randomUUID();
        final String description = name + "-Description";


        final DatasetInfo dataset = importer.createViewFromCsv(name, description, CRIMES_CSV_HEADER);
        TestCase.assertNotNull(dataset);
        TestCase.assertNotNull(dataset.getId());

        UpsertResult result = producer.upsertCsv(dataset.getId(), CRIMES_CSV_UPSERT);
        TestCase.assertNotNull(result);
View Full Code Here

    }

    @Test
    public void testUpsertWithRowIdentifier() throws IOException, SodaError, InterruptedException {
        final Soda2Producer producer = createProducer();
        final SodaImporter importer = createImporter();

        final String name = "RowIdUpsert" + UUID.randomUUID();
        final String description = name + "-Description";

        //Import a CSV and set the rowIdentifier to be ID
        final DatasetInfo dataset = importer.createViewFromCsv(name, description, CRIMES_CSV_HEADER, "ID");
        TestCase.assertNotNull(dataset);
        TestCase.assertNotNull(dataset.getId());
        importer.publish(dataset.getId());

        try {

            //
            //Verify the row we expect is really there.
            final SoqlQuery   lookupTestRow = new SoqlQueryBuilder()
                        .setWhereClause("id='8880962'")
                        .build();
            final List queryResults = producer.query(dataset.getId(), lookupTestRow, Soda2Producer.HASH_RETURN_TYPE);
            TestCase.assertEquals(1, queryResults.size());

            final Map result = (Map) queryResults.get(0);
            TestCase.assertEquals("8880962", result.get("id"));
            TestCase.assertEquals("THEFT", result.get("primary_type"));


            //
            //  Update the dataset by uploading a CSV stream
            final InputStream  csvStream = getClass().getResourceAsStream("/testCrimesHeader2.csv");
            final UpsertResult results = producer.upsertStream(dataset.getId(), HttpLowLevel.CSV_TYPE, csvStream);
            TestCase.assertEquals(1, results.getRowsCreated());
            TestCase.assertEquals(0, results.errorCount());
            TestCase.assertEquals(0, results.getRowsDeleted());
            TestCase.assertEquals(2, results.getRowsUpdated());

            //
            //   Verify an overwrite happened, and not just an append.
            final List queryResults2 = producer.query(dataset.getId(), lookupTestRow, Soda2Producer.HASH_RETURN_TYPE);
            TestCase.assertEquals(1, queryResults.size());

            final Map result2 = (Map) queryResults2.get(0);
            TestCase.assertEquals("8880962", result2.get("id"));
            TestCase.assertEquals("BATTERY", result2.get("primary_type"));

            //
            //  Test adding a stream that has an invalid row in it
            final InputStream  csvStreamInvalid = getClass().getResourceAsStream("/testCrimesWithInvalidCrime.csv");
            final UpsertResult resultsInvalid = producer.upsertStream(dataset.getId(), HttpLowLevel.CSV_TYPE, csvStreamInvalid);
            TestCase.assertEquals(0, resultsInvalid.getRowsCreated());
            TestCase.assertEquals(1, resultsInvalid.errorCount());
            TestCase.assertEquals(0, resultsInvalid.getRowsDeleted());
            TestCase.assertEquals(2, resultsInvalid.getRowsUpdated());

            TestCase.assertEquals(1, resultsInvalid.getErrors().get(0).getIndex());
            TestCase.assertEquals("", resultsInvalid.getErrors().get(0).getPrimaryKey());


        } finally {
            importer.deleteDataset(dataset.getId());
        }

    }
View Full Code Here

                    } else {
                        // attach a requestId to all Producer API calls (for error tracking purposes)
                        String jobRequestId = Utils.generateRequestId();
                        producer = Soda2Producer.newProducerWithRequestId(
                                connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken(), jobRequestId);
                        final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken());
                        int filesizeChunkingCutoffBytes = userPrefs.getFilesizeChunkingCutoffMB() == null ? 10 * NUM_BYTES_PER_MB :
                                Integer.parseInt(userPrefs.getFilesizeChunkingCutoffMB()) * NUM_BYTES_PER_MB;
                        int numRowsPerChunk = userPrefs.getNumRowsPerChunk() == null ? 10000 :
                                Integer.parseInt(userPrefs.getNumRowsPerChunk());
                        switch (publishMethod) {
View Full Code Here

TOP

Related Classes of com.socrata.api.SodaImporter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.