Package com.socrata.model.importer

Examples of com.socrata.model.importer.DatasetInfo


    }

    @Override
    public DatasetInfo build()
    {
        final DatasetInfo retVal = new Dataset();
        populate(retVal);
        return retVal;
    }
View Full Code Here


        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, TEST_NOMINATIONA_CSV, "Name");
        try {
            //Get the ID column
            int columnId = ((Dataset)datasetCreated).getRowIdentifierColumnId();
            sodaImporter.removeColumn(datasetCreated.getId(), columnId);


            try {
                sodaImporter.append(datasetCreated.getId(), TEST_NOMINATIONA_CSV, 1, null);
                TestCase.fail("Expected failure after updating a non-existing column as a rowidentifier");
            } catch (MetadataUpdateError mue) {
                 //Success error expected
            }

        } finally {
            sodaImporter.deleteDataset(datasetCreated.getId());
        }

    }
View Full Code Here

        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, WIDE_CSV, "col0");
        try {
            //Get the ID column
            int columnId = ((Dataset)datasetCreated).getRowIdentifierColumnId();
            sodaImporter.removeColumn(datasetCreated.getId(), columnId);

            /*
            try {
                sodaImporter.append(datasetCreated.getId(), WIDE_CSV, 1, null);
                TestCase.fail("Expected failure after updating a non-existing column as a rowidentifier");
            } catch (MetadataUpdateError mue) {
                //Success error expected
            }
            */
        } finally {
            sodaImporter.deleteDataset(datasetCreated.getId());
        }

    }
View Full Code Here

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());

        try {

            List<Crime> results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            UpsertResult replaceResult = producer.replaceCsv(datasetPublished.getId(), CRIMES_HEADER_CSV);
            TestCase.assertEquals(2, replaceResult.getRowsCreated());
            TestCase.assertEquals(0, replaceResult.errorCount());
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(2, results.size());

            replaceResult = producer.replace(datasetPublished.getId(), Lists.newArrayList(results.get(0)));
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());


            //Try it with a larger file
            /*
            replaceResult = producer.replaceCsv(datasetPublished.getId(), CRIMES_APPEND_CSV);
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1000, results.size());
            */

        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
        }

    }
View Full Code Here

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        FileInputStream fisCrimes1 = new FileInputStream(CRIMES_HEADER_CSV);
        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());

        try {

            List<Crime> results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            UpsertResult replaceResult = producer.replaceStream(datasetPublished.getId(), HttpLowLevel.CSV_TYPE, fisCrimes1);
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(2, results.size());

            replaceResult = producer.replace(datasetPublished.getId(), Lists.newArrayList(results.get(0)));
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());


        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
            GeneralUtils.closeQuietly(fisCrimes1);
        }

    }
View Full Code Here

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());
        DatasetInfo datasetWorkingCopy = sodaImporter.createWorkingCopy(datasetPublished.getId());

        try {

            List<Crime> results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            UpsertResult replaceResult = producer.replaceCsv(datasetWorkingCopy.getId(), CRIMES_HEADER_CSV);
            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(2, results.size());

            producer.upsertCsv(datasetWorkingCopy.getId(), CRIMES_HEADER2_CSV);
            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());

            /*
            replaceResult = producer.upsertCsv(datasetWorkingCopy.getId(), CRIMES_APPEND_CSV);
            TestCase.assertEquals(3, replaceResult.getRowsUpdated());
            TestCase.assertEquals(997, replaceResult.getRowsCreated());

            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1000, results.size());
            */

            replaceResult = producer.replace(datasetWorkingCopy.getId(), Lists.newArrayList(results.get(0)));
            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());

            datasetPublished = sodaImporter.publish(datasetWorkingCopy.getId());
        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
        }

    }
View Full Code Here

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());

        try {

            List<Crime> results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            //Delete two crimes using upsert
            UpsertResult replaceResult = producer.upsertCsv(datasetPublished.getId(), DELETE_CRIMES_CSV);
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());

            producer.delete(datasetPublished.getId(), results.get(0).getId().toString());
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(0, results.size());


        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
        }

    }
View Full Code Here


    @Test
    public void testNoUserId() throws IOException
    {
        DatasetInfo datasetInfo =  mapper.readValue(VIEW_W_PERMISSIONS, DatasetInfo.class);
        TestCase.assertNotNull(datasetInfo);
        TestCase.assertNotNull(datasetInfo.getGrants());
        TestCase.assertNotNull(datasetInfo.getTableAuthor());
        TestCase.assertEquals("8910-1234", datasetInfo.getTableAuthor().getId());
        TestCase.assertEquals("Table Author", datasetInfo.getTableAuthor().getDisplayName());
        TestCase.assertEquals("Table Author Screen Name", datasetInfo.getTableAuthor().getScreenName());


        TestCase.assertNotNull(datasetInfo.getOwner());
        TestCase.assertEquals("1234-5678", datasetInfo.getOwner().getId());
        TestCase.assertEquals("Test Owner", datasetInfo.getOwner().getDisplayName());
        TestCase.assertEquals("Test Owner Screen Name", datasetInfo.getOwner().getScreenName());



    }
View Full Code Here

        final String name = "LongUpsertName" + UUID.randomUUID();
        final String description = name + "-Description";


        final DatasetInfo dataset = importer.createViewFromCsv(name, description, CRIMES_CSV_HEADER);
        TestCase.assertNotNull(dataset);
        TestCase.assertNotNull(dataset.getId());

        UpsertResult result = producer.upsertCsv(dataset.getId(), CRIMES_CSV_UPSERT);
        TestCase.assertNotNull(result);
        TestCase.assertEquals(0, result.errorCount());
        TestCase.assertEquals(4999, result.getRowsCreated());
        TestCase.assertEquals(0, result.getRowsUpdated());
    }
View Full Code Here

        final String name = "RowIdUpsert" + UUID.randomUUID();
        final String description = name + "-Description";

        //Import a CSV and set the rowIdentifier to be ID
        final DatasetInfo dataset = importer.createViewFromCsv(name, description, CRIMES_CSV_HEADER, "ID");
        TestCase.assertNotNull(dataset);
        TestCase.assertNotNull(dataset.getId());
        importer.publish(dataset.getId());

        try {

            //
            //Verify the row we expect is really there.
            final SoqlQuery   lookupTestRow = new SoqlQueryBuilder()
                        .setWhereClause("id='8880962'")
                        .build();
            final List queryResults = producer.query(dataset.getId(), lookupTestRow, Soda2Producer.HASH_RETURN_TYPE);
            TestCase.assertEquals(1, queryResults.size());

            final Map result = (Map) queryResults.get(0);
            TestCase.assertEquals("8880962", result.get("id"));
            TestCase.assertEquals("THEFT", result.get("primary_type"));


            //
            //  Update the dataset by uploading a CSV stream
            final InputStream  csvStream = getClass().getResourceAsStream("/testCrimesHeader2.csv");
            final UpsertResult results = producer.upsertStream(dataset.getId(), HttpLowLevel.CSV_TYPE, csvStream);
            TestCase.assertEquals(1, results.getRowsCreated());
            TestCase.assertEquals(0, results.errorCount());
            TestCase.assertEquals(0, results.getRowsDeleted());
            TestCase.assertEquals(2, results.getRowsUpdated());

            //
            //   Verify an overwrite happened, and not just an append.
            final List queryResults2 = producer.query(dataset.getId(), lookupTestRow, Soda2Producer.HASH_RETURN_TYPE);
            TestCase.assertEquals(1, queryResults.size());

            final Map result2 = (Map) queryResults2.get(0);
            TestCase.assertEquals("8880962", result2.get("id"));
            TestCase.assertEquals("BATTERY", result2.get("primary_type"));

            //
            //  Test adding a stream that has an invalid row in it
            final InputStream  csvStreamInvalid = getClass().getResourceAsStream("/testCrimesWithInvalidCrime.csv");
            final UpsertResult resultsInvalid = producer.upsertStream(dataset.getId(), HttpLowLevel.CSV_TYPE, csvStreamInvalid);
            TestCase.assertEquals(0, resultsInvalid.getRowsCreated());
            TestCase.assertEquals(1, resultsInvalid.errorCount());
            TestCase.assertEquals(0, resultsInvalid.getRowsDeleted());
            TestCase.assertEquals(2, resultsInvalid.getRowsUpdated());

            TestCase.assertEquals(1, resultsInvalid.getErrors().get(0).getIndex());
            TestCase.assertEquals("", resultsInvalid.getErrors().get(0).getPrimaryKey());


        } finally {
            importer.deleteDataset(dataset.getId());
        }

    }
View Full Code Here

TOP

Related Classes of com.socrata.model.importer.DatasetInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.