Package com.socrata.api

Source Code of com.socrata.api.Soda2ProducerTest

package com.socrata.api;

import com.google.common.collect.Lists;
import com.socrata.TestBase;
import com.socrata.exceptions.LongRunningQueryException;
import com.socrata.exceptions.SodaError;
import com.socrata.model.UpsertResult;
import com.socrata.model.importer.DatasetInfo;
import com.socrata.model.soql.SoqlQuery;
import com.socrata.utils.GeneralUtils;
import junit.framework.TestCase;
import org.junit.Test;
import test.model.Crime;

import java.io.*;
import java.util.List;
import java.util.UUID;

/**
*/
public class Soda2ProducerTest extends TestBase
{

    public static final File  CRIMES_HEADER_CSV = new File("src/test/resources/testCrimesHeader.csv");
    public static final File  CRIMES_HEADER2_CSV = new File("src/test/resources/testCrimesHeader2.csv");
    public static final File  CRIMES_APPEND_CSV = new File("src/test/resources/testCrimesAppend.csv");

    public static final File  DELETE_CRIMES_CSV = new File("src/test/resources/testDeletingCrimes.csv");


    /**
     * Tests a number of ways to connect using dataset name
     */
    @Test
    public void testReplace() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());

        try {

            List<Crime> results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            UpsertResult replaceResult = producer.replaceCsv(datasetPublished.getId(), CRIMES_HEADER_CSV);
            TestCase.assertEquals(2, replaceResult.getRowsCreated());
            TestCase.assertEquals(0, replaceResult.errorCount());
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(2, results.size());

            replaceResult = producer.replace(datasetPublished.getId(), Lists.newArrayList(results.get(0)));
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());


            //Try it with a larger file
            /*
            replaceResult = producer.replaceCsv(datasetPublished.getId(), CRIMES_APPEND_CSV);
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1000, results.size());
            */

        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
        }

    }

    /**
     * Tests a number of ways to connect using dataset name
     */
    @Test
    public void testReplaceStream() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        FileInputStream fisCrimes1 = new FileInputStream(CRIMES_HEADER_CSV);
        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());

        try {

            List<Crime> results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            UpsertResult replaceResult = producer.replaceStream(datasetPublished.getId(), HttpLowLevel.CSV_TYPE, fisCrimes1);
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(2, results.size());

            replaceResult = producer.replace(datasetPublished.getId(), Lists.newArrayList(results.get(0)));
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());


        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
            GeneralUtils.closeQuietly(fisCrimes1);
        }

    }

    @Test
    public void testSoda2OnWorkingCopy() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {

        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());
        DatasetInfo datasetWorkingCopy = sodaImporter.createWorkingCopy(datasetPublished.getId());

        try {

            List<Crime> results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            UpsertResult replaceResult = producer.replaceCsv(datasetWorkingCopy.getId(), CRIMES_HEADER_CSV);
            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(2, results.size());

            producer.upsertCsv(datasetWorkingCopy.getId(), CRIMES_HEADER2_CSV);
            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());

            /*
            replaceResult = producer.upsertCsv(datasetWorkingCopy.getId(), CRIMES_APPEND_CSV);
            TestCase.assertEquals(3, replaceResult.getRowsUpdated());
            TestCase.assertEquals(997, replaceResult.getRowsCreated());

            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1000, results.size());
            */

            replaceResult = producer.replace(datasetWorkingCopy.getId(), Lists.newArrayList(results.get(0)));
            results = producer.query(datasetWorkingCopy.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());

            datasetPublished = sodaImporter.publish(datasetWorkingCopy.getId());
        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
        }

    }

    /**
     * Tests a number of ways to connect using dataset name
     */
    @Test
    public void testDelete() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final String name = "Name" + UUID.randomUUID();
        final String description = name + "-Description";

        final HttpLowLevel  connection = connect();
        final Soda2Producer producer = new Soda2Producer(connection);
        final SodaImporter  sodaImporter = new SodaImporter(connection);

        DatasetInfo datasetCreated = sodaImporter.createViewFromCsv(name, description, CRIMES_HEADER2_CSV, "ID");
        DatasetInfo datasetPublished = sodaImporter.publish(datasetCreated.getId());

        try {

            List<Crime> results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(3, results.size());


            //Delete two crimes using upsert
            UpsertResult replaceResult = producer.upsertCsv(datasetPublished.getId(), DELETE_CRIMES_CSV);
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(1, results.size());

            producer.delete(datasetPublished.getId(), results.get(0).getId().toString());
            results = producer.query(datasetPublished.getId(), SoqlQuery.SELECT_ALL, Crime.LIST_TYPE);
            TestCase.assertEquals(0, results.size());


        } finally {
            sodaImporter.deleteDataset(datasetPublished.getId());
        }

    }


    public static final String UPSERT_RESULT_NO_ERRORS = "{ \"rows_created\":1, \"rows_updated\":2, \"rows_deleted\":3 }";
    public static final String UPSERT_RESULT_1 = "{ \"rows_created\":1, \"rows_updated\":2, \"rows_deleted\":3, \"errors\":[{\"error\":\"Error1\", \"input_index\":1, \"primary_key\":\"key1\"}] }";
    public static final String UPSERT_RESULT_2 = "{ \"rows_created\":1, \"rows_updated\":2, \"rows_deleted\":3, \"errors\":[{\"error\":\"Error1\", \"input_index\":1, \"primary_key\":\"key1\"},{\"error\":\"Error2\", \"input_index\":2, \"primary_key\":\"key2\"}] }";

    @Test
    public void testParsingUpsertResults() throws IOException
    {

        UpsertResult noErrors = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(UPSERT_RESULT_NO_ERRORS.getBytes("utf-8")));
        TestCase.assertEquals(0, noErrors.errorCount());
        TestCase.assertEquals(1, noErrors.getRowsCreated());
        TestCase.assertEquals(2, noErrors.getRowsUpdated());
        TestCase.assertEquals(3, noErrors.getRowsDeleted());

        UpsertResult errors1 = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(UPSERT_RESULT_1.getBytes("utf-8")));
        TestCase.assertEquals(1, errors1.errorCount());
        TestCase.assertEquals(1, errors1.getErrors().get(0).getIndex());
        TestCase.assertEquals("Error1", errors1.getErrors().get(0).getError());
        TestCase.assertEquals("key1", errors1.getErrors().get(0).getPrimaryKey());
        TestCase.assertEquals(1, errors1.getRowsCreated());
        TestCase.assertEquals(2, errors1.getRowsUpdated());
        TestCase.assertEquals(3, errors1.getRowsDeleted());

        UpsertResult errors2 = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(UPSERT_RESULT_2.getBytes("utf-8")));
        TestCase.assertEquals(2, errors2.errorCount());
        TestCase.assertEquals(1, errors2.getErrors().get(0).getIndex());
        TestCase.assertEquals("Error1", errors2.getErrors().get(0).getError());
        TestCase.assertEquals("key1", errors2.getErrors().get(0).getPrimaryKey());
        TestCase.assertEquals(2, errors2.getErrors().get(1).getIndex());
        TestCase.assertEquals("Error2", errors2.getErrors().get(1).getError());
        TestCase.assertEquals("key2", errors2.getErrors().get(1).getPrimaryKey());
        TestCase.assertEquals(1, errors2.getRowsCreated());
        TestCase.assertEquals(2, errors2.getRowsUpdated());
        TestCase.assertEquals(3, errors2.getRowsDeleted());
    }


    public static final String SODA_SERVER_UPSERT_RESULT_NO_ERRORS = "[{\"typ\":\"insert\", \"id\":\"key1\", \"ver\":\"1\"}, {\"typ\":\"insert\", \"id\":\"key2\", \"ver\":\"1\"}, {\"typ\":\"insert\", \"id\":\"key3\", \"ver\":\"1\" }]";
    public static final String SODA_SERVER_UPSERT_RESULT_1 =         "[{\"typ\":\"update\", \"id\":\"key1\", \"ver\":\"1\"}, {\"typ\":\"update\", \"id\":\"key2\", \"ver\":\"1\"}, {\"typ\":\"error\", \"id\":\"key3\", \"ver\":\"1\", \"err\":\"error3\" }]";
    public static final String SODA_SERVER_UPSERT_RESULT_2 =         "[{\"typ\":\"delete\", \"id\":\"key1\", \"ver\":\"1\"}, {\"typ\":\"error\",  \"id\":\"key2\", \"ver\":\"1\", \"err\":\"error2\"}, {\"typ\":\"error\", \"id\":\"key3\", \"ver\":\"1\", \"err\":\"error3\" }]";

    @Test
    public void testParsingSodaServerUpsertResults() throws IOException
    {

        UpsertResult noErrors = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(SODA_SERVER_UPSERT_RESULT_NO_ERRORS.getBytes("utf-8")));
        TestCase.assertEquals(0, noErrors.errorCount());
        TestCase.assertEquals(3, noErrors.getRowsCreated());
        TestCase.assertEquals(0, noErrors.getRowsUpdated());
        TestCase.assertEquals(0, noErrors.getRowsDeleted());

        UpsertResult errors1 = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(SODA_SERVER_UPSERT_RESULT_1.getBytes("utf-8")));
        TestCase.assertEquals(1, errors1.errorCount());
        TestCase.assertEquals("error3", errors1.getErrors().get(0).getError());
        TestCase.assertEquals("key3", errors1.getErrors().get(0).getPrimaryKey());
        TestCase.assertEquals(0, errors1.getRowsCreated());
        TestCase.assertEquals(2, errors1.getRowsUpdated());
        TestCase.assertEquals(0, errors1.getRowsDeleted());

        UpsertResult errors2 = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(SODA_SERVER_UPSERT_RESULT_2.getBytes("utf-8")));
        TestCase.assertEquals(2, errors2.errorCount());
        TestCase.assertEquals("error2", errors2.getErrors().get(0).getError());
        TestCase.assertEquals("key2", errors2.getErrors().get(0).getPrimaryKey());
        TestCase.assertEquals("error3", errors2.getErrors().get(1).getError());
        TestCase.assertEquals("key3", errors2.getErrors().get(1).getPrimaryKey());
        TestCase.assertEquals(0, errors2.getRowsCreated());
        TestCase.assertEquals(0, errors2.getRowsUpdated());
        TestCase.assertEquals(1, errors2.getRowsDeleted());
    }


    public static final String SODA_SERVER_UPSERT_RESULT_NO_ERRORS_OLD = "[[{\"typ\":\"insert\", \"id\":\"key1\", \"ver\":\"1\"}, {\"typ\":\"insert\", \"id\":\"key2\", \"ver\":\"1\"}, {\"typ\":\"insert\", \"id\":\"key3\", \"ver\":\"1\" }]]";
    public static final String SODA_SERVER_UPSERT_RESULT_1_OLD =         "[[{\"typ\":\"update\", \"id\":\"key1\", \"ver\":\"1\"}, {\"typ\":\"update\", \"id\":\"key2\", \"ver\":\"1\"}, {\"typ\":\"error\",  \"id\":\"key3\", \"ver\":\"1\", \"err\":\"error3\" }]]";
    public static final String SODA_SERVER_UPSERT_RESULT_2_OLD =         "[[{\"typ\":\"delete\", \"id\":\"key1\", \"ver\":\"1\"}, {\"typ\":\"error\",  \"id\":\"key2\", \"ver\":\"1\", \"err\":\"error2\"}, {\"typ\":\"error\", \"id\":\"key3\", \"ver\":\"1\", \"err\":\"error3\" }]]";

    @Test
    public void testParsingOldSodaServerUpsertResults() throws IOException
    {

        UpsertResult noErrors = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(SODA_SERVER_UPSERT_RESULT_NO_ERRORS_OLD.getBytes("utf-8")));
        TestCase.assertEquals(0, noErrors.errorCount());
        TestCase.assertEquals(3, noErrors.getRowsCreated());
        TestCase.assertEquals(0, noErrors.getRowsUpdated());
        TestCase.assertEquals(0, noErrors.getRowsDeleted());

        UpsertResult errors1 = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(SODA_SERVER_UPSERT_RESULT_1_OLD.getBytes("utf-8")));
        TestCase.assertEquals(1, errors1.errorCount());
        TestCase.assertEquals("error3", errors1.getErrors().get(0).getError());
        TestCase.assertEquals("key3", errors1.getErrors().get(0).getPrimaryKey());
        TestCase.assertEquals(0, errors1.getRowsCreated());
        TestCase.assertEquals(2, errors1.getRowsUpdated());
        TestCase.assertEquals(0, errors1.getRowsDeleted());

        UpsertResult errors2 = Soda2Producer.deserializeUpsertResult(new ByteArrayInputStream(SODA_SERVER_UPSERT_RESULT_2_OLD.getBytes("utf-8")));
        TestCase.assertEquals(2, errors2.errorCount());
        TestCase.assertEquals("error2", errors2.getErrors().get(0).getError());
        TestCase.assertEquals("key2", errors2.getErrors().get(0).getPrimaryKey());
        TestCase.assertEquals("error3", errors2.getErrors().get(1).getError());
        TestCase.assertEquals("key3", errors2.getErrors().get(1).getPrimaryKey());
        TestCase.assertEquals(0, errors2.getRowsCreated());
        TestCase.assertEquals(0, errors2.getRowsUpdated());
        TestCase.assertEquals(1, errors2.getRowsDeleted());
    }
}
TOP

Related Classes of com.socrata.api.Soda2ProducerTest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.