Package com.socrata.api

Examples of com.socrata.api.Soda2Producer


    protected Soda2Consumer createConsumer() throws IOException {
        return new Soda2Consumer(connect());
    }

    protected Soda2Producer createProducer() throws IOException {
        return new Soda2Producer(connect());
    }
View Full Code Here


     * Will issue a simple query, and do spot validation.
     */
    @Test
    public void testCsvUpload() throws LongRunningQueryException, SodaError, InterruptedException, IOException
    {
        final Soda2Producer producer = createProducer();
        final InputStream  csvStream = getClass().getResourceAsStream("/testNominations.csv");

        final UpsertResult results = producer.upsertStream(UPDATE_DATA_SET, HttpLowLevel.CSV_TYPE, csvStream);
        TestCase.assertEquals(2, results.getRowsCreated());
        TestCase.assertEquals(0, results.errorCount());
        TestCase.assertEquals(0, results.getRowsDeleted());
        TestCase.assertEquals(0, results.getRowsUpdated());

View Full Code Here

    @Test
    public void testLongUpsert() throws IOException, SodaError, InterruptedException
    {

        final Soda2Producer producer = createProducer();
        final SodaImporter importer = createImporter();

        final String name = "LongUpsertName" + UUID.randomUUID();
        final String description = name + "-Description";


        final DatasetInfo dataset = importer.createViewFromCsv(name, description, CRIMES_CSV_HEADER);
        TestCase.assertNotNull(dataset);
        TestCase.assertNotNull(dataset.getId());

        UpsertResult result = producer.upsertCsv(dataset.getId(), CRIMES_CSV_UPSERT);
        TestCase.assertNotNull(result);
        TestCase.assertEquals(0, result.errorCount());
        TestCase.assertEquals(4999, result.getRowsCreated());
        TestCase.assertEquals(0, result.getRowsUpdated());
    }
View Full Code Here

        TestCase.assertEquals(0, result.getRowsUpdated());
    }

    @Test
    public void testUpsertWithRowIdentifier() throws IOException, SodaError, InterruptedException {
        final Soda2Producer producer = createProducer();
        final SodaImporter importer = createImporter();

        final String name = "RowIdUpsert" + UUID.randomUUID();
        final String description = name + "-Description";

        //Import a CSV and set the rowIdentifier to be ID
        final DatasetInfo dataset = importer.createViewFromCsv(name, description, CRIMES_CSV_HEADER, "ID");
        TestCase.assertNotNull(dataset);
        TestCase.assertNotNull(dataset.getId());
        importer.publish(dataset.getId());

        try {

            //
            //Verify the row we expect is really there.
            final SoqlQuery   lookupTestRow = new SoqlQueryBuilder()
                        .setWhereClause("id='8880962'")
                        .build();
            final List queryResults = producer.query(dataset.getId(), lookupTestRow, Soda2Producer.HASH_RETURN_TYPE);
            TestCase.assertEquals(1, queryResults.size());

            final Map result = (Map) queryResults.get(0);
            TestCase.assertEquals("8880962", result.get("id"));
            TestCase.assertEquals("THEFT", result.get("primary_type"));


            //
            //  Update the dataset by uploading a CSV stream
            final InputStream  csvStream = getClass().getResourceAsStream("/testCrimesHeader2.csv");
            final UpsertResult results = producer.upsertStream(dataset.getId(), HttpLowLevel.CSV_TYPE, csvStream);
            TestCase.assertEquals(1, results.getRowsCreated());
            TestCase.assertEquals(0, results.errorCount());
            TestCase.assertEquals(0, results.getRowsDeleted());
            TestCase.assertEquals(2, results.getRowsUpdated());

            //
            //   Verify an overwrite happened, and not just an append.
            final List queryResults2 = producer.query(dataset.getId(), lookupTestRow, Soda2Producer.HASH_RETURN_TYPE);
            TestCase.assertEquals(1, queryResults.size());

            final Map result2 = (Map) queryResults2.get(0);
            TestCase.assertEquals("8880962", result2.get("id"));
            TestCase.assertEquals("BATTERY", result2.get("primary_type"));

            //
            //  Test adding a stream that has an invalid row in it
            final InputStream  csvStreamInvalid = getClass().getResourceAsStream("/testCrimesWithInvalidCrime.csv");
            final UpsertResult resultsInvalid = producer.upsertStream(dataset.getId(), HttpLowLevel.CSV_TYPE, csvStreamInvalid);
            TestCase.assertEquals(0, resultsInvalid.getRowsCreated());
            TestCase.assertEquals(1, resultsInvalid.errorCount());
            TestCase.assertEquals(0, resultsInvalid.getRowsDeleted());
            TestCase.assertEquals(2, resultsInvalid.getRowsUpdated());
View Full Code Here

        final Date voteDate = new Date();

        final Nomination  nomination = new Nomination(UUID.randomUUID().toString(), "Test Agent", "Department of Testing", "https://sandbox.demo.socrata.com", nominationDate, null, false, false);
        final Nomination  nominationUpdate = new Nomination(nomination.getName(), "Test Agent", "Department of Testing", "https://sandbox.demo.socrata.com", nominationDate, voteDate, true, false);

        final Soda2Producer producer= createProducer();

        final Meta objectMetadata = producer.addObject(UPDATE_DATA_SET, nomination);
        final Nomination createdNomination = producer.getById(UPDATE_DATA_SET, objectMetadata.getId(), Nomination.class);
        TestCase.assertTrue(EqualsBuilder.reflectionEquals(nomination, createdNomination));

        final Meta updateMeta= producer.update(UPDATE_DATA_SET, objectMetadata.getId(), nominationUpdate);
        TestCase.assertEquals(objectMetadata.getId(), updateMeta.getId());

        TestCase.assertEquals(objectMetadata.getCreatedAt(), updateMeta.getCreatedAt());
        TestCase.assertEquals(objectMetadata.getCreatedMeta(), updateMeta.getCreatedMeta());
        TestCase.assertFalse(objectMetadata.getCreatedAt().after(updateMeta.getUpdatedAt()));

        final Nomination updatedNomination = producer.getById(UPDATE_DATA_SET, objectMetadata.getId(), Nomination.class);

        //UNDONE(willpugh) -- Turn back on when bug 7102 is fixed.
        //TestCase.assertTrue(EqualsBuilder.reflectionEquals(nominationUpdate, updatedNomination));

        List l = Lists.newArrayList(new DeleteRecord(objectMetadata.getId(), true));
        ObjectMapper m = new ObjectMapper();
        System.out.println(m.writeValueAsString(l));
        UpsertResult result = producer.upsert(UPDATE_DATA_SET, l);
        try {
            final Nomination deletedNomination = producer.getById(UPDATE_DATA_SET, objectMetadata.getId(), Nomination.class);
            TestCase.fail();
        } catch (DoesNotExistException e) {
            //Expected Condition
        }
View Full Code Here

      // streamExporter "exports" the source dataset rows
      final Soda2Consumer streamExporter = Soda2Consumer.newConsumer(
          sourceSiteDomain, connectionInfo.getUser(),
          connectionInfo.getPassword(), connectionInfo.getToken());
      // streamUpserter "upserts" the rows exported to the created dataset
      final Soda2Producer streamUpserter = Soda2Producer.newProducer(
          sinkSiteDomain, connectionInfo.getUser(),
          connectionInfo.getPassword(), portDestinationDomainAppToken);
      String errorMessage = "";
      boolean noPortExceptions = false;
      try {
View Full Code Here

  }

    //Probably makes sense to make one generic addLogEntry() for all job types
    public static JobStatus addLogEntry(String logDatasetID, SocrataConnectionInfo connectionInfo,
                                        MetadataJob job, JobStatus status) {
        final Soda2Producer producer = Soda2Producer.newProducer(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken());

        List<Map<String, Object>> upsertObjects = new ArrayList<Map<String, Object>>();
        Map<String, Object> newCols = new HashMap<String,Object>();

        // add standard log data
        Date currentDateTime = new Date();
        newCols.put("Date", (Object) currentDateTime);
        newCols.put("DatasetID", (Object) job.getDatasetID());
        newCols.put("JobFile", (Object) job.getPathToSavedFile());
        if(status.isError()) {
            newCols.put("Errors", (Object) status.getMessage());
        } else {
            newCols.put("Success", (Object) true);
        }
        upsertObjects.add(ImmutableMap.copyOf(newCols));

        JobStatus logStatus = JobStatus.SUCCESS;
        String errorMessage = "";
        boolean noPublishExceptions = false;
        try {
            producer.upsert(logDatasetID, upsertObjects);
            noPublishExceptions = true;
        }
        catch (SodaError sodaError) {
            errorMessage = sodaError.getMessage();
        }
View Full Code Here

        } else {
            JobStatus validationStatus = IntegrationJobValidity.validateJobParams(connectionInfo, this);
            if (validationStatus.isError()) {
                runStatus = validationStatus;
            } else {
                Soda2Producer producer = null;
                try {
                    File fileToPublishFile = new File(fileToPublish);
                    if (publishViaDi2Http) {
                        try (DeltaImporter2Publisher publisher = new DeltaImporter2Publisher(userPrefs)) {
                            // "upsert" == "append" in di2
                            if ("upsert".equalsIgnoreCase(controlFile.action))
                                controlFile.action = "Append";
                            // TODO: remove the next line when di2 is updated to accept lowercase variants
                            controlFile.action = Utils.capitalizeFirstLetter(controlFile.action);
                            runStatus = publisher.publishWithDi2OverHttp(datasetID, fileToPublishFile, controlFile);
                        }
                    } else if (publishViaFTP) {
                        runStatus = doPublishViaFTPv2(fileToPublishFile);
                    } else {
                        // attach a requestId to all Producer API calls (for error tracking purposes)
                        String jobRequestId = Utils.generateRequestId();
                        producer = Soda2Producer.newProducerWithRequestId(
                                connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken(), jobRequestId);
                        final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken());
                        int filesizeChunkingCutoffBytes = userPrefs.getFilesizeChunkingCutoffMB() == null ? 10 * NUM_BYTES_PER_MB :
                                Integer.parseInt(userPrefs.getFilesizeChunkingCutoffMB()) * NUM_BYTES_PER_MB;
                        int numRowsPerChunk = userPrefs.getNumRowsPerChunk() == null ? 10000 :
                                Integer.parseInt(userPrefs.getNumRowsPerChunk());
                        switch (publishMethod) {
                            case upsert:
                            case append:
                                result = doAppendOrUpsertViaHTTP(
                                        producer, importer, fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk);
                                break;
                            case replace:
                                result = Soda2Publisher.replaceNew(
                                        producer, importer, datasetID, fileToPublishFile, fileToPublishHasHeaderRow);
                                break;
                            case delete:
                                result = doDeleteViaHTTP(
                                        producer, importer, fileToPublishFile, filesizeChunkingCutoffBytes, numRowsPerChunk);
                                break;
                            default:
                                runStatus = JobStatus.INVALID_PUBLISH_METHOD;
                        }
                    }

                } catch (IOException | SodaError | InterruptedException e) {
                    publishExceptions = e.getMessage();
                    e.printStackTrace();
                } finally {
                    if (producer != null) producer.close();
                }
            }
        }

        if (publishExceptions.length() > 0) {
View Full Code Here

     *
     * @return null if log entry was added successfully, otherwise return an error message as a String
     */
    public static String addLogEntry(final String logDatasetID, final SocrataConnectionInfo connectionInfo,
                                     final IntegrationJob job, final JobStatus status, final UpsertResult result) {
        final Soda2Producer producer = Soda2Producer.newProducer(connectionInfo.getUrl(), connectionInfo.getUser(),
                connectionInfo.getPassword(), connectionInfo.getToken());

        List<Map<String, Object>> upsertObjects = new ArrayList<>();
        Map<String, Object> newCols = new HashMap<>();

        // add standard log data
        Date currentDateTime = new Date();
        newCols.put("Date", currentDateTime);
        newCols.put("DatasetID", job.getDatasetID());
        newCols.put("FileToPublish", job.getFileToPublish());
        if(job.getPublishMethod() != null)
            newCols.put("PublishMethod", job.getPublishMethod());
        newCols.put("JobFile", job.getPathToSavedFile());
        if(result != null) {
            newCols.put("RowsUpdated", result.rowsUpdated);
            newCols.put("RowsCreated", result.rowsCreated);
            newCols.put("RowsDeleted", result.rowsDeleted);
        } else {
            newCols.put("RowsUpdated", (status.rowsUpdated == null ? 0 : status.rowsUpdated));
            newCols.put("RowsCreated", (status.rowsCreated == null ? 0 : status.rowsCreated));
            newCols.put("RowsDeleted", (status.rowsDeleted == null ? 0 : status.rowsDeleted));
        }
        if(status.isError()) {
            newCols.put("Errors", status.getMessage());
        } else {
            newCols.put("Success", true);
        }
        newCols.put("DataSyncVersion", VersionProvider.getThisVersion());
        upsertObjects.add(ImmutableMap.copyOf(newCols));

        String logPublishingErrorMessage = null;
        try {
            producer.upsert(logDatasetID, upsertObjects);
        }
        catch (SodaError | InterruptedException e) {
            e.printStackTrace();
            logPublishingErrorMessage = e.getMessage();
        }
View Full Code Here

*/
public class IntegrationUtilityTest extends TestBase {

    @Test
    public void testReplaceNew() throws LongRunningQueryException, SodaError, IOException, InterruptedException {
        final Soda2Producer producer = createProducer();
        final SodaDdl ddl = createSodaDdl();

        File twoRowsFile = new File("src/test/resources/datasync_unit_test_two_rows.csv");
        UpsertResult result = Soda2Publisher.replaceNew(producer, ddl, UNITTEST_DATASET_ID, twoRowsFile, true);

View Full Code Here

TOP

Related Classes of com.socrata.api.Soda2Producer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.