Package com.socrata.model.importer

Examples of com.socrata.model.importer.Dataset

22
23
24
25
26
27
28
29
30
    }

    @Override
    public DatasetInfo build()
    {
        final DatasetInfo retVal = new Dataset();
        populate(retVal);
        return retVal;
    }
View Full Code Here
47
48
49
50
51
52
53
54
    public void testViewMapping() throws IOException
    {
        final JacksonObjectMapperProvider objectMapperProvider = new JacksonObjectMapperProvider();
        final ObjectMapper  mapper = objectMapperProvider.getContext(null);

        Dataset v = mapper.readValue(new File("src/test/resources/view.json"), Dataset.class);
        TestCase.assertNotNull(v);
    }
View Full Code Here
557
558
559
560
561
562
563
564
565
566
567
         * @throws com.socrata.exceptions.SodaError
         * @throws InterruptedException
         */
        private String generateControlFileContent(String domain, String fileToPublish, PublishMethod publishMethod,
                                                  String datasetId, boolean containsHeaderRow) throws IOException, URISyntaxException, HttpException {
            Dataset datasetInfo = DatasetUtils.getDatasetInfo(domain, datasetId);
            boolean useGeocoding = DatasetUtils.hasLocationColumn(datasetInfo);

            String[] columns = null;
            if (!containsHeaderRow) {
                if (PublishMethod.delete.equals(publishMethod))
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
  public static String portSchema(SodaDdl loader, SodaDdl creator,
      final String sourceSetID, final String destinationDatasetTitle,
            final boolean useNewBackend) throws SodaError, InterruptedException {
    System.out.print("Copying schema from dataset " + sourceSetID);
        Dataset sourceSet = (Dataset) loader.loadDatasetInfo(sourceSetID);
        if(destinationDatasetTitle != null && !destinationDatasetTitle.equals(""))
            sourceSet.setName(destinationDatasetTitle);

        adaptSchemaForAggregates(sourceSet);

        // TODO uncomment (after soda-java is updated to support this)
    //DatasetInfo sinkSet = creator.createDataset(sourceSet, useNewBackend);
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
    }

    public static JobStatus assertSchemasAreAlike(SodaDdl sourceChecker, SodaDdl sinkChecker, String sourceSetID, String sinkSetID)
            throws SodaError, InterruptedException {
        // We don't need to test metadata; we're only concerned with the columns...
        Dataset sourceSchema = (Dataset) sourceChecker.loadDatasetInfo(sourceSetID);
        Dataset sinkSchema = (Dataset) sinkChecker.loadDatasetInfo(sinkSetID);
        // Grab the columns...
        List<Column> sourceColumns = sourceSchema.getColumns();
        List<Column> sinkColumns = sinkSchema.getColumns();
        // And let the tests begin.
        if(sourceColumns.size() == sinkColumns.size()) {
            // If the sizes are the same we can begin comparing columns
            for (int i = 0; i < sourceColumns.size(); i++) {
                // The aspects of the columns that we care about are the API field names and their data types
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
        String fileExtension = Utils.getFileExtension(fileToPublish);
        if(!allowedFileToPublishExtensions.contains(fileExtension))
            return JobStatus.FILE_TO_PUBLISH_INVALID_FORMAT;

        Dataset schema;
        try {
            schema = DatasetUtils.getDatasetInfo(connectionInfo.getUrl(), job.getDatasetID());

            if(job.getPublishViaDi2Http() || job.getPublishViaFTP()) {
View Full Code Here
71
72
73
74
75
76
77
78
     * @param domain
     * @param datasetId
     * @return list of field names or null if there
     */
    public static String getFieldNamesString(String domain, String datasetId) throws HttpException, IOException, URISyntaxException {
        Dataset datasetInfo = getDatasetInfo(domain, datasetId);
        return getFieldNamesString(datasetInfo);
    }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
                    headers[i] = headers[i].trim();
                }
            }
        } else {
            // get API field names for each column in dataset
            Dataset info = (Dataset) ddl.loadDatasetInfo(id);
            List<Column> columns = info.getColumns();
            headers = new String[columns.size()];
            for(int i = 0; i < columns.size(); i++) {
                headers[i] = columns.get(i).getFieldName();
            }
        }
View Full Code Here
229
230
231
232
233
234
235
236
237
238
239
240
            throws SodaError, InterruptedException, IOException {
        return publishViaSoda2(producer, ddl, PublishMethod.replace, id, file, 0, containsHeaderRow);
    }

    private static String getDatasetRowId(SodaDdl ddl, String id) throws SodaError, InterruptedException {
        Dataset info = (Dataset) ddl.loadDatasetInfo(id);
        Column rowIdentifier = info.lookupRowIdentifierColumn();
        String rowIdentifierName;
        if (rowIdentifier == null) {
            rowIdentifierName = ":id";
        } else {
            rowIdentifierName = rowIdentifier.getFieldName();
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
            TestCase.assertEquals(sourceMeta.getCategory(), sinkMeta.getCategory());
            TestCase.assertEquals(sourceMeta.getTags(), sinkMeta.getTags());
            TestCase.assertEquals(sourceMeta.getRights(), sinkMeta.getRights());

            // Next, test the schema by grabbing Dataset and Column objects from DatasetInfo.
            Dataset sourceSchema = (Dataset) sourceMeta;
            Dataset sinkSchema = (Dataset) sinkMeta;
            List<Column> sourceColumns = sourceSchema.getColumns();
            List<Column> sinkColumns = sinkSchema.getColumns();

            TestCase.assertEquals(sourceColumns.size(), sinkColumns.size());
            for (int i = 0; i < sourceColumns.size(); i++) {
                TestCase.assertEquals(sourceColumns.get(i).getName(), sinkColumns.get(i).getName());
                TestCase.assertEquals(sourceColumns.get(i).getFieldName(), sinkColumns.get(i).getFieldName());
View Full Code Here
TOP

Related Classes of com.socrata.model.importer.Dataset

  • com.socrata.builders.DatasetInfoBuilder
  • com.socrata.datasync.DatasetUtils
  • com.socrata.datasync.PortUtility
  • com.socrata.datasync.publishers.Soda2Publisher
  • com.socrata.datasync.ui.IntegrationJobTab$EditGenerateControlFileListener
  • com.socrata.datasync.utilities.IntegrationUtilityTest
  • com.socrata.datasync.utilities.PortUtilityTest
  • com.socrata.datasync.validation.IntegrationJobValidity
  • com.socrata.utils.TestJacksonObjectMapperProvider
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.