Package com.socrata.api

Source Code of com.socrata.api.SodaImporterTest

package com.socrata.api;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.socrata.TestBase;
import com.socrata.builders.BlueprintBuilder;
import com.socrata.builders.ExternalDatasetBuilder;
import com.socrata.builders.NonDataFileDatasetBuilder;
import com.socrata.builders.SoqlQueryBuilder;
import com.socrata.exceptions.LongRunningQueryException;
import com.socrata.exceptions.SodaError;
import com.socrata.model.Address;
import com.socrata.model.GeocodingResults;
import com.socrata.model.importer.*;
import com.socrata.model.soql.OrderByClause;
import com.socrata.model.soql.SoqlQuery;
import com.socrata.model.soql.SortOrder;
import junit.framework.TestCase;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Test;
import test.model.Nomination;
import test.model.NominationsWText;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;

/**
*/
public class SodaImporterTest extends TestBase
{

    public static final File  NOMINATIONS_CSV = new File("src/test/resources/testNominations.csv");
    public static final File  CRIMES_CSV = new File("src/test/resources/testCrimes.csv");
    public static final File  CRIMES_HEADER_CSV = new File("src/test/resources/testCrimesHeader.csv");
    public static final File  BABY_NAMES_LOC = new File("src/test/resources/testBabyNames.csv");
    public static final File  BABY_NAMES_LOC_2 = new File("src/test/resources/testBabyNames2.csv");
    public static final File  BABY_NAMES_LOC_3 = new File("src/test/resources/testBabyNames3.csv");
    public static final File  BRONX_ADMIN_BOUNDARY = new File("src/test/resources/BronxAdminBoundaryLayers.zip");


    /**
     * Tests a number of ways to connect using dataset name
     */
    @Test
    public void testImport() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);
        final DatasetInfo createdView = importer.createViewFromCsv(name, description, NOMINATIONS_CSV);

        try {
            TestCase.assertNotNull(createdView);
            TestCase.assertNotNull(createdView.getId());
            TestCase.assertEquals(name, createdView.getName());
            TestCase.assertEquals(description, createdView.getDescription());
            TestCase.assertEquals("unpublished", createdView.getPublicationStage());

            GeocodingResults results = importer.findPendingGeocodingResults(createdView.getId());
            TestCase.assertEquals(0, results.getView());

            importer.waitForPendingGeocoding(createdView.getId());
            importer.publish(createdView.getId());

            DatasetInfo loadedView = importer.loadDatasetInfo(createdView.getId());
            TestCase.assertNotNull(loadedView);
            TestCase.assertEquals("published", loadedView.getPublicationStage());

            final Soda2Consumer consumer = new Soda2Consumer(connection);
            final SoqlQuery   sortQuery = new SoqlQueryBuilder()
                    .addOrderByPhrase(new OrderByClause(SortOrder.Ascending, "name"))
                    .build();
            List<NominationsWText> nominations = consumer.query(createdView.getId(), sortQuery, NominationsWText.LIST_TYPE);
            TestCase.assertEquals(2, nominations.size());
            TestCase.assertEquals("Kitty, Hello", nominations.get(0).getName());
            TestCase.assertEquals("Name, Test", nominations.get(1).getName());
        } finally {
            importer.deleteDataset(createdView.getId());
        }
    }

    @Test
    public void testImportWithSetResourceName() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);

        DatasetInfo createdView = importer.createViewFromCsv(name, description, NOMINATIONS_CSV);

        try {
            //
            //  Make sure data is added
            Soda2Consumer consumer = new Soda2Consumer(connection);
            SoqlQuery   sortQuery = new SoqlQueryBuilder()
                    .addOrderByPhrase(new OrderByClause(SortOrder.Ascending, "name"))
                    .build();
            List<NominationsWText> nominations = consumer.query(createdView.getId(), sortQuery, NominationsWText.LIST_TYPE);
            TestCase.assertEquals(2, nominations.size());
            TestCase.assertEquals("Kitty, Hello", nominations.get(0).getName());
            TestCase.assertEquals("Name, Test", nominations.get(1).getName());



            createdView.setResourceName(name);
            createdView = importer.updateDatasetInfo(createdView);

            consumer = new Soda2Consumer(connection);
            sortQuery = new SoqlQueryBuilder()
                    .addOrderByPhrase(new OrderByClause(SortOrder.Ascending, "name"))
                    .build();
            nominations = consumer.query(createdView.getId(), sortQuery, NominationsWText.LIST_TYPE);
            TestCase.assertEquals(2, nominations.size());
            TestCase.assertEquals("Kitty, Hello", nominations.get(0).getName());
            TestCase.assertEquals("Name, Test", nominations.get(1).getName());

        } finally {
            importer.deleteDataset(createdView.getId());
        }
    }



    @Test
    public void testWithTypes() throws IOException, InterruptedException, SodaError, LongRunningQueryException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);
        final ScanResults  scanResults = importer.scan(NOMINATIONS_CSV);
        final Blueprint    blueprint = new BlueprintBuilder()
                .setName(name)
                .setDescription(description)
                .setSkip(1)
                .addColumn(new BlueprintColumn("Name", "Name Description", "Text"))
                .addColumn(new BlueprintColumn("Position", "Position Description", "Text"))
                .addColumn(new BlueprintColumn("Agency Name", "Agency Name Description", "Text"))
                .addColumn(new BlueprintColumn("Agency Website", "Agency Website Description", "Text"))
                .addColumn(new BlueprintColumn("Nomination Date", "Nomination Date Description", "calendar_date"))
                .addColumn(new BlueprintColumn("Confirmation Vote", "Confirmation Vote Description", "calendar_date"))
                .addColumn(new BlueprintColumn("Confirmed", "Confirmed Description", "checkbox"))
                .addColumn(new BlueprintColumn("Holdover", "Holdover Description", "checkbox"))
                .build();

        final DatasetInfo createdView = importer.importScanResults(blueprint, null, NOMINATIONS_CSV, scanResults);



        //final Dataset createdView = importer.createViewFromCsv(name, description, NOMINATIONS_CSV);

        try {
            TestCase.assertNotNull(createdView);
            TestCase.assertNotNull(createdView.getId());
            TestCase.assertEquals(name, createdView.getName());
            TestCase.assertEquals(description, createdView.getDescription());
            TestCase.assertEquals("unpublished", createdView.getPublicationStage());

            GeocodingResults results = importer.findPendingGeocodingResults(createdView.getId());
            TestCase.assertEquals(0, results.getView());

            importer.waitForPendingGeocoding(createdView.getId());
            importer.publish(createdView.getId());

            Dataset loadedView = (Dataset) importer.loadDatasetInfo(createdView.getId());
            TestCase.assertNotNull(loadedView);
            TestCase.assertEquals("published", loadedView.getPublicationStage());

            final Soda2Consumer consumer = new Soda2Consumer(connection);
            final SoqlQuery   sortQuery = new SoqlQueryBuilder()
                    .addOrderByPhrase(new OrderByClause(SortOrder.Ascending, "name"))
                    .build();
            List<Nomination> nominations = consumer.query(createdView.getId(), sortQuery, Nomination.LIST_TYPE);
            TestCase.assertEquals(2, nominations.size());
            TestCase.assertEquals("Kitty, Hello", nominations.get(0).getName());
            TestCase.assertEquals("Name, Test", nominations.get(1).getName());
        } finally {
            importer.deleteDataset(createdView.getId());
        }
    }

    @Test
    public void testSimpleCreate() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);


        final Dataset view = new Dataset();
        view.setName(name);
        view.setDescription("Hello Kitty");
        view.setTags(Lists.newArrayList("Red", "Blue"));
        view.setColumns(new ArrayList<Column>());
        view.setFlags(new ArrayList<String>());

        final Dataset createdView = (Dataset) importer.createDataset(view);
        TestCase.assertNotNull(createdView);
        TestCase.assertNotNull(createdView.getId());
        TestCase.assertEquals("unpublished", createdView.getPublicationStage());

        importer.publish(createdView.getId());

        createdView.setDescription("New Description");
        final Dataset updatedView = (Dataset) importer.updateDatasetInfo(createdView);
        TestCase.assertEquals("New Description", updatedView.getDescription());
        TestCase.assertEquals("published", updatedView.getPublicationStage());

        Soda2Producer   producer = new Soda2Producer(connection);
        FileInputStream fis = new FileInputStream(NOMINATIONS_CSV);
        producer.upsertStream(updatedView.getId(), HttpLowLevel.CSV_TYPE, fis);
        fis.close();

        Dataset loadedView = (Dataset) importer.loadDatasetInfo(createdView.getId());
        TestCase.assertNotNull(loadedView);
        TestCase.assertEquals("published", loadedView.getPublicationStage());

        importer.deleteDataset(createdView.getId());
    }


    @Test
    public void testSettingPrimaryKey() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);

        Map<String, String> format = new HashMap<>();
        format.put("noCommas", "true");

        final Dataset view = new Dataset();
        view.setName(name);
        view.setDescription("Hello Kitty");
        view.setTags(Lists.newArrayList("Red", "Blue"));
        view.setColumns(Lists.newArrayList(
                new Column(0, "col1", "col1", "col1-desc", "Text", 0, 10, format, "Text"),
                new Column(0, "col2", "col2", "col2-desc", "Text", 0, 10, format, "Text")
        ));
        view.setFlags(new ArrayList<String>());

        final Dataset createdView = (Dataset) importer.createDataset(view);
        TestCase.assertNotNull(createdView);
        TestCase.assertNotNull(createdView.getId());
        TestCase.assertEquals("unpublished", createdView.getPublicationStage());


        final Dataset loadView = (Dataset) importer.loadDatasetInfo(createdView.getId());
        TestCase.assertEquals(name, loadView.getName());
        TestCase.assertEquals(2, loadView.getColumns().size());


        //Set license
        loadView.setLicenseId("CC_30_BY_NC");
        loadView.setLicense(new License("Creative Commons Attribution-Noncommercial 3.0 Unported", "images/licenses/cc30bync.png", "http://creativecommons.org/licenses/by-nc/3.0/legalcode"));
        loadView.setAttribution("Socrata Test");
        loadView.setAttributionLink("https://www.socrata.com");

        //Set Metadata
        loadView.setMetadata(new Metadata(ImmutableMap.<String, Map<String, String>>of("geostuff", ImmutableMap.<String, String>of("stuff", "bax")),
                                                                          "col1", null, null, null));
        loadView.setRowIdentifierColumnId(loadView.getColumns().get(0).getId());

        loadView.setCategory("Fun");
        loadView.setExternalId("ExternalId_For_Me");

        //Update dataset
        final Dataset updateView2 = (Dataset) importer.updateDatasetInfo(loadView);
        final Dataset loadView2 = (Dataset) importer.loadDatasetInfo(createdView.getId());

        TestCase.assertEquals(loadView.getLicenseId(), loadView2.getLicenseId());
        TestCase.assertEquals(loadView.getLicense().getName(), loadView2.getLicense().getName());
        TestCase.assertEquals(loadView.getLicense().getLogoUrl(), loadView2.getLicense().getLogoUrl());
        TestCase.assertEquals(loadView.getLicense().getTermsLink(), loadView2.getLicense().getTermsLink());

        TestCase.assertEquals(loadView.getCategory(), loadView2.getCategory());
        TestCase.assertEquals(loadView.getExternalId(), loadView2.getExternalId());

        TestCase.assertEquals(loadView.getRowIdentifierColumnId(),       loadView2.getRowIdentifierColumnId());


        TestCase.assertEquals(loadView.getMetadata().getCustom_fields().get("geostuff").get("stuff"), loadView2.getMetadata().getCustom_fields().get("geostuff").get("stuff"));

        TestCase.assertEquals(loadView.getAttribution(), loadView2.getAttribution());
        TestCase.assertEquals(loadView.getAttributionLink(), loadView2.getAttributionLink());

        final Dataset updateView3 = (Dataset) importer.updateDatasetInfo(loadView2);
        final Dataset loadView3 = (Dataset) importer.loadDatasetInfo(createdView.getId());
        TestCase.assertEquals(loadView.getLicenseId(), loadView3.getLicenseId());
        TestCase.assertEquals(loadView.getLicense().getName(), loadView3.getLicense().getName());
        TestCase.assertEquals(loadView.getLicense().getLogoUrl(), loadView3.getLicense().getLogoUrl());
        TestCase.assertEquals(loadView.getLicense().getTermsLink(), loadView3.getLicense().getTermsLink());

        TestCase.assertEquals(loadView.getCategory(), loadView3.getCategory());
        TestCase.assertEquals(loadView.getExternalId(), loadView3.getExternalId());
        TestCase.assertEquals(loadView2.getMetadata().getRowIdentifier(), loadView3.getMetadata().getRowIdentifier());
        TestCase.assertEquals(loadView.getMetadata().getCustom_fields().get("geostuff").get("stuff"), loadView3.getMetadata().getCustom_fields().get("geostuff").get("stuff"));

        TestCase.assertEquals(loadView.getAttribution(), loadView3.getAttribution());
        TestCase.assertEquals(loadView.getAttributionLink(), loadView3.getAttributionLink());

        importer.deleteDataset(createdView.getId());
    }

    @Test
    public void testAttachments() throws LongRunningQueryException, SodaError, InterruptedException, IOException {

        final String name = "Name" + UUID.randomUUID();

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);

        Map<String, String> format = new HashMap<>();
        format.put("noCommas", "true");

        final Dataset view = new Dataset();
        view.setName(name);
        view.setDescription("Hello Kitty");
        view.setTags(Lists.newArrayList("Red", "Blue"));
        view.setColumns(Lists.newArrayList(
                new Column(0, "col1", "col1", "col1-desc", "Text", 0, 10, format, "Text"),
                new Column(0, "col2", "col2", "col2-desc", "Text", 0, 10, format, "Text")
        ));
        view.setFlags(new ArrayList<String>());

        final Dataset createdView = (Dataset) importer.createDataset(view);
        TestCase.assertNotNull(createdView);
        TestCase.assertNotNull(createdView.getId());
        TestCase.assertEquals("unpublished", createdView.getPublicationStage());


        //
        //  Test Adding and getting the Assets
        final AssetResponse response = importer.addAsset(NOMINATIONS_CSV);
        TestCase.assertNotNull(response);
        TestCase.assertNotNull(response.getId());

        final InputStream inputStream = importer.getAsset(response.getId());
        TestCase.assertNotNull(inputStream);

        final String returnedAsset = IOUtils.toString(inputStream);
        final String originalAsset = FileUtils.readFileToString(NOMINATIONS_CSV);
        TestCase.assertEquals(returnedAsset, originalAsset);

        //
        //  Test using assets for attachement
        final Metadata    metadata = new Metadata(null, null, null, null, Lists.newArrayList(new Attachment(response.getId(), response.getNameForOutput(), response.getNameForOutput())));
        final Dataset loadView = (Dataset) importer.loadDatasetInfo(createdView.getId());
        loadView.setMetadata(metadata);
        importer.updateDatasetInfo(loadView);

        final Dataset loadView2 = (Dataset) importer.loadDatasetInfo(createdView.getId());
        final Metadata loadedMetadata = loadView2.getMetadata();
        TestCase.assertEquals(1, loadedMetadata.getAttachments().size());
        TestCase.assertEquals(response.getId(), loadedMetadata.getAttachments().get(0).getBlobId());


    }

    @Test
    public void testImportWithPK() throws InterruptedException, SodaError, IOException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);
        final Soda2Consumer consumer = new Soda2Consumer(connection);
        final Soda2Producer producer = new Soda2Producer(connection);
        final DatasetInfo createdView = importer.createViewFromCsv(name, description, NOMINATIONS_CSV, "Name");

        try {
            importer.publish(createdView.getId());

            SoqlQuery   sortByNameQuery = new SoqlQueryBuilder(SoqlQuery.SELECT_ALL)
                    .addOrderByPhrase(new OrderByClause(SortOrder.Descending, "name"))
                    .build();

            //Validate
            List retVal =  consumer.query(createdView.getId(), sortByNameQuery, Soda2Consumer.HASH_RETURN_TYPE);
            TestCase.assertEquals(2, retVal.size());
            TestCase.assertEquals("Name, Test", ((Map)retVal.get(0)).get("name"));
            TestCase.assertEquals("TEST (http://www.test.com)", ((Map) retVal.get(0)).get("agency_website"));

            //Test an update that will fail because of a PK constraint
            Nomination duplicatePk = new Nomination("Name, Test", "New Position", "This Agency", "http://foo.bar", new Date(), new Date(), false, false);

            producer.addObject(createdView.getId(), duplicatePk);
            //Verify the existing record was updated, rather than a new one being created
            retVal =  consumer.query(createdView.getId(), sortByNameQuery, Soda2Consumer.HASH_RETURN_TYPE);
            TestCase.assertEquals(2, retVal.size());
            TestCase.assertEquals("Name, Test", ((Map)retVal.get(0)).get("name"));
            TestCase.assertEquals("http://foo.bar", ((Map)retVal.get(0)).get("agency_website"));



        } finally {
            importer.deleteDataset(createdView.getId());
        }


    }

    @Test
    public void testCreatingFileDataset() throws IOException, SodaError, InterruptedException
    {
        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);

        final String name = "Name" + UUID.randomUUID();
        final String description = "Description" + name;


        final NonDataFileDataset fileDataset = importer.importNonDataFile(name, description, NOMINATIONS_CSV);

        try {
            TestCase.assertNotNull(fileDataset);
            TestCase.assertEquals(name, fileDataset.getName());
            TestCase.assertEquals(description, fileDataset.getDescription());
            TestCase.assertEquals("testNominations.csv", fileDataset.getBlobFilename());
            TestCase.assertEquals(NOMINATIONS_CSV.length(), fileDataset.getBlobFileSize());
            TestCase.assertEquals("published", fileDataset.getPublicationStage());

            importer.makePublic(fileDataset.getId());
            final NonDataFileDataset fileDatasetLoaded = (NonDataFileDataset) importer.loadDatasetInfo(fileDataset.getId());
            TestCase.assertNotNull(fileDatasetLoaded);
            TestCase.assertEquals(name, fileDatasetLoaded.getName());
            TestCase.assertEquals(description, fileDatasetLoaded.getDescription());
            TestCase.assertEquals("testNominations.csv", fileDatasetLoaded.getBlobFilename());
            TestCase.assertEquals(NOMINATIONS_CSV.length(), fileDatasetLoaded.getBlobFileSize());
            TestCase.assertEquals("published", fileDatasetLoaded.getPublicationStage());

            String fileUploadContent = IOUtils.toString(importer.getFileBlob(fileDatasetLoaded));
            String fileContent = FileUtils.readFileToString(NOMINATIONS_CSV);
            TestCase.assertEquals(fileContent, fileUploadContent);



            final NonDataFileDatasetBuilder nonDataFileDatasetBuilder = new NonDataFileDatasetBuilder(fileDatasetLoaded)
                    .addTag("TestFile")
                    .setLicenseId("CC_30_BY_NC")
                    .setLicense(new License("Creative Commons Attribution-Noncommercial 3.0 Unported", "images/licenses/cc30bync.png", "http://creativecommons.org/licenses/by-nc/3.0/legalcode"))
                    .setAttribution("Socrata Test")
                    .setAttributionLink("https://www.socrata.com");

            final NonDataFileDataset fileDatasetLoaded2 = (NonDataFileDataset) importer.updateDatasetInfo(nonDataFileDatasetBuilder.build());
            TestCase.assertEquals("CC_30_BY_NC", fileDatasetLoaded2.getLicenseId());
            TestCase.assertEquals("Socrata Test", fileDatasetLoaded2.getAttribution());
            TestCase.assertEquals("https://www.socrata.com", fileDatasetLoaded2.getAttributionLink());


            final NonDataFileDataset newUploadedFile = importer.replaceNonDataFile(fileDatasetLoaded.getId(), BABY_NAMES_LOC);
            TestCase.assertEquals("CC_30_BY_NC", newUploadedFile.getLicenseId());
            TestCase.assertEquals("Socrata Test", newUploadedFile.getAttribution());
            TestCase.assertEquals("https://www.socrata.com", newUploadedFile.getAttributionLink());
            TestCase.assertFalse(fileDatasetLoaded2.getBlobId().equals(newUploadedFile.getBlobId()));

            String newFileUploadContent = IOUtils.toString(importer.getFileBlob(newUploadedFile));
            String newFileContent = FileUtils.readFileToString(BABY_NAMES_LOC);
            TestCase.assertEquals(newFileContent, newFileUploadContent);


        } finally {
            importer.deleteDataset(fileDataset.getId());
        }

    }


    @Test
    public void testCreatingExternalDataset() throws IOException, SodaError, InterruptedException
    {
        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);

        final String name = "Name" + UUID.randomUUID();
        final String description = "Description" + name;


        final ExternalDataset    dataset1 = new ExternalDatasetBuilder()
                .setMetadata(new Metadata())
                .addAccessPoint("html", "http://www.google.com")
                .setName(name)
                .setDescription(description)
                .build();

        final ExternalDataset fileDataset = (ExternalDataset) importer.createDataset(dataset1);

        try {
            TestCase.assertNotNull(fileDataset);
            TestCase.assertEquals(name, fileDataset.getName());
            TestCase.assertEquals(description, fileDataset.getDescription());
            TestCase.assertEquals("http://www.google.com", fileDataset.getMetadata().getAccessPoints().get("html"));
            TestCase.assertEquals("unpublished", fileDataset.getPublicationStage());

            importer.publish(fileDataset.getId());
            importer.makePublic(fileDataset.getId());

            final ExternalDataset fileDatasetLoaded = (ExternalDataset) importer.loadDatasetInfo(fileDataset.getId());
            TestCase.assertNotNull(fileDatasetLoaded);
            TestCase.assertEquals(name, fileDatasetLoaded.getName());
            TestCase.assertEquals(description, fileDatasetLoaded.getDescription());
            TestCase.assertEquals("http://www.google.com", fileDatasetLoaded.getMetadata().getAccessPoints().get("html"));
            TestCase.assertEquals("published", fileDatasetLoaded.getPublicationStage());

            final ExternalDatasetBuilder externalDatasetBuilder = new ExternalDatasetBuilder(fileDatasetLoaded)
                    .addTag("TestFile")
                    .setLicenseId("CC_30_BY_NC")
                    .setLicense(new License("Creative Commons Attribution-Noncommercial 3.0 Unported", "images/licenses/cc30bync.png", "http://creativecommons.org/licenses/by-nc/3.0/legalcode"))
                    .setAttribution("Socrata Test")
                    .setAttributionLink("https://www.socrata.com");

            final ExternalDataset fileDatasetLoaded2 = (ExternalDataset) importer.updateDatasetInfo(externalDatasetBuilder.build());
            TestCase.assertEquals("CC_30_BY_NC", fileDatasetLoaded2.getLicenseId());
            TestCase.assertEquals("Socrata Test", fileDatasetLoaded2.getAttribution());
            TestCase.assertEquals("https://www.socrata.com", fileDatasetLoaded2.getAttributionLink());
            TestCase.assertEquals(name, fileDatasetLoaded2.getName());
            TestCase.assertEquals(description, fileDatasetLoaded2.getDescription());
            TestCase.assertEquals("http://www.google.com", fileDatasetLoaded2.getMetadata().getAccessPoints().get("html"));
            TestCase.assertEquals("published", fileDatasetLoaded2.getPublicationStage());

        } finally {
            importer.deleteDataset(fileDataset.getId());
        }

    }

    //@Test
    public void testWithLocations() throws IOException, InterruptedException, SodaError
    {

        final String name = "Testing Loc" ;

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);
        final Soda2Consumer consumer= new Soda2Consumer(connection);

        ScanResults scanResults = importer.scan(BABY_NAMES_LOC);


        final Blueprint blueprint = new BlueprintBuilder()
                .setName(name)
                .setDescription("Description")
                .setSkip(1)
                .addColumn(new BlueprintColumn("first_name","First Name",   "Text"))
                .addColumn(new BlueprintColumn("county",    "County",       "Text"))
                .addColumn(new BlueprintColumn("sex",       "Sex",          "Number"))
                .addColumn(new BlueprintColumn("count",     "count",        "Number"))
                .addColumn(new BlueprintColumn("rollnumber","RollNumber",   "Number"))
                .addColumn(new BlueprintColumn("street",    "Street",       "Text"))
                .addColumn(new BlueprintColumn("city",      "City",         "Text"))
                .addColumn(new BlueprintColumn("state",     "State",        "Text"))
                .addColumn(new BlueprintColumn("zipcode",   "Zip Code",     "Text"))
                .addColumn(new BlueprintColumn("location""location",     "location"))
                .build();


        final String[] translation = new String[] {"col1", "col2","col3",
                "col4","col5","col6","col7","col8","col9", "'(' + col6 + ',' + col7 + ',' + col8 + ',' + col9 + ')'"};
        DatasetInfo dataset = importer.importScanResults(blueprint, translation, BABY_NAMES_LOC, scanResults);

        dataset = importer.publish(dataset.getId());

        //Verify the address got setup correctly
        List results = consumer.query(dataset.getId(), SoqlQuery.SELECT_ALL, Soda2Consumer.HASH_RETURN_TYPE);
        ObjectMapper mapper = new ObjectMapper();
        TestCase.assertEquals(4, results.size());
        for (Object curr : results) {
            Map currRow = (Map)curr;
            Map location = (Map) currRow.get("location");
            Map human_address = (Map) location.get("human_address");

            Address address = new Address((String)human_address.get("address"),
                                          (String)human_address.get("city"),
                                          (String)human_address.get("state"),
                                          (String)human_address.get("zip"));

            TestCase.assertNotNull(location.get("longitude"));
            TestCase.assertNotNull(location.get("latitude"));
            TestCase.assertEquals(currRow.get("street"), address.getStreetAddress());
            TestCase.assertEquals(currRow.get("city"), address.getCity());
            TestCase.assertEquals(currRow.get("state"), address.getState());
            TestCase.assertEquals(currRow.get("zipcode"), address.getZip());
        }


        importer.deleteDataset(dataset.getId());
    }

    //@Test
    public void testWithLocations2() throws IOException, InterruptedException, SodaError
    {

        final String name = "Testing Loc" ;

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);
        final Soda2Consumer consumer= new Soda2Consumer(connection);
        ScanResults scanResults = importer.scan(BABY_NAMES_LOC_2);


        final Blueprint blueprint = new BlueprintBuilder()
                .setName(name)
                .setDescription("Description")
                .setSkip(1)
                .addColumn(new BlueprintColumn("first_name","First Name",   "Text"))
                .addColumn(new BlueprintColumn("county",    "County",       "Text"))
                .addColumn(new BlueprintColumn("sex",       "Sex",          "Number"))
                .addColumn(new BlueprintColumn("count",     "count",        "Number"))
                .addColumn(new BlueprintColumn("rollnumber","RollNumber",   "Number"))
                .addColumn(new BlueprintColumn("address",    "Address",     "Location"))
                .build();


        DatasetInfo dataset = importer.importScanResults(blueprint, null, BABY_NAMES_LOC_2, scanResults);
        importer.publish(dataset.getId());

        //Verify the address got setup correctly
        List results = consumer.query(dataset.getId(), SoqlQuery.SELECT_ALL, Soda2Consumer.HASH_RETURN_TYPE);
        ObjectMapper mapper = new ObjectMapper();
        TestCase.assertEquals(4, results.size());
        for (Object curr : results) {
            Map currRow = (Map)curr;
            Map location = (Map) currRow.get("address");
            Map human_address = (Map) location.get("human_address");

            Address address = new Address((String)human_address.get("address"),
                                          (String)human_address.get("city"),
                                          (String)human_address.get("state"),
                                          (String)human_address.get("zip"));

            TestCase.assertNotNull(location.get("longitude"));
            TestCase.assertNotNull(location.get("latitude"));
            TestCase.assertNotNull(address.getStreetAddress());
            TestCase.assertNotNull(address.getCity());
            TestCase.assertNotNull(address.getState());
            TestCase.assertNotNull(address.getZip());
        }

        importer.deleteDataset(dataset.getId());
    }

    //@Test
    public void testWithLocations3() throws IOException, InterruptedException, SodaError
    {

        final String name = "Testing Loc" ;

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);
        final Soda2Consumer consumer= new Soda2Consumer(connection);

        ScanResults scanResults = importer.scan(BABY_NAMES_LOC_3);


        final Blueprint blueprint = new BlueprintBuilder()
                .setName(name)
                .setDescription("Description")
                .setSkip(1)
                .addColumn(new BlueprintColumn("first_name", "First Name", "Text"))
                .addColumn(new BlueprintColumn("county",    "County",       "Text"))
                .addColumn(new BlueprintColumn("sex",       "Sex",          "Number"))
                .addColumn(new BlueprintColumn("count",     "count",        "Number"))
                .addColumn(new BlueprintColumn("rollnumber","RollNumber",   "Number"))
                .addColumn(new BlueprintColumn("lat",       "Lat",          "Number"))
                .addColumn(new BlueprintColumn("long",      "Long",         "Number"))
                .addColumn(new BlueprintColumn("location""location",     "location"))
                .build();


        final String[] translation = new String[] {"col1", "col2","col3","col4","col5","col6", "col7","'(' + col6 + ',' + col7 + ')'"};
        DatasetInfo dataset = importer.importScanResults(blueprint, translation, BABY_NAMES_LOC_3, scanResults);

        importer.publish(dataset.getId());

        //Verify the address got setup correctly
        List results = consumer.query(dataset.getId(), SoqlQuery.SELECT_ALL, Soda2Consumer.HASH_RETURN_TYPE);
        TestCase.assertEquals(4, results.size());
        for (Object curr : results) {
            Map currRow = (Map)curr;
            Map location = (Map) currRow.get("location");
            TestCase.assertNotNull(currRow.get("lat"));
            TestCase.assertNotNull(currRow.get("long"));
            TestCase.assertNotNull(location.get("longitude"));
            TestCase.assertNotNull(location.get("latitude"));
        }

        importer.deleteDataset(dataset.getId());
    }


    @Test
    public void testWithShapefile() throws IOException, InterruptedException, SodaError
    {

        final HttpLowLevel connection = connect();
        final SodaImporter importer = new SodaImporter(connection);

        final GeoDataset dataset = (GeoDataset) importer.createViewFromShapefile(BRONX_ADMIN_BOUNDARY);
        TestCase.assertNotNull(dataset);

        try {
            GeoInfo  geoInfo = dataset.getMetadata().getGeo();
            TestCase.assertNotNull(geoInfo);
            TestCase.assertNotNull(geoInfo.namespace);
            TestCase.assertEquals(7, geoInfo.decodeLayers().length);
            TestCase.assertNotNull(geoInfo.owsUrl);
            TestCase.assertEquals(4, geoInfo.decodeBbox().length);


            final GeoDataset replacedDataset = (GeoDataset) importer.replaceViewFromShapefile(dataset.getId(), BRONX_ADMIN_BOUNDARY);
            TestCase.assertNotNull(replacedDataset);
            TestCase.assertTrue(dataset.getCreatedAt().equals(replacedDataset.getCreatedAt()));
            TestCase.assertFalse(dataset.getViewLastModified().equals(replacedDataset.getViewLastModified()));

        } finally {
            importer.deleteDataset(dataset.getId());
        }
    }

}
TOP

Related Classes of com.socrata.api.SodaImporterTest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.