Package org.apache.falcon.entity

Examples of org.apache.falcon.entity.CatalogStorage

558
559
560
561
562
563
564
565
566
567
568
569
570
                propagateFileSystemCopyProperties(pathsWithPartitions, props);
            } else if (sourceStorage.getType() == Storage.TYPE.TABLE) {
                instancePaths = "${coord:dataIn('input')}";

                final CatalogStorage sourceTableStorage = (CatalogStorage) sourceStorage;
                propagateTableStorageProperties(srcCluster, sourceTableStorage, props, "falconSource");
                final CatalogStorage targetTableStorage = (CatalogStorage) targetStorage;
                propagateTableStorageProperties(trgCluster, targetTableStorage, props, "falconTarget");
                propagateTableCopyProperties(srcCluster, sourceTableStorage,
                    trgCluster, targetTableStorage, props);
                setupHiveConfiguration(srcCluster, trgCluster, wfPath);
            }
View Full Code Here
201
202
203
204
205
206
207
208
209
        }
    }

    private String getTableFeedInstanceName(Feed feed, String feedInstancePath,
                                            Storage.TYPE storageType) throws URISyntaxException {
        CatalogStorage instanceStorage = (CatalogStorage) FeedHelper.createStorage(
                storageType.name(), feedInstancePath);
        return feed.getName() + "/" + instanceStorage.toPartitionAsPath();
    }
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
     * @throws FalconException
     */
    private long getTablePartitionCreateTimeMetric(String feedUriTemplate)
        throws IOException, URISyntaxException, FalconException {

        CatalogStorage storage = (CatalogStorage)
                FeedHelper.createStorage(Storage.TYPE.TABLE.name(), feedUriTemplate);
        CatalogPartition partition = CatalogServiceFactory.getCatalogService().getPartition(
                storage.getCatalogUrl(), storage.getDatabase(), storage.getTable(), storage.getPartitions());
        return partition == null ? 0 : partition.getCreateTime();
    }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
        Storage storage = FeedHelper.createStorage(cluster, feed);
        if (storage.getType() == Storage.TYPE.FILESYSTEM) {  // FS does NOT use staging dirs
            return;
        }

        final CatalogStorage tableStorage = (CatalogStorage) storage;
        String stagingDir = FeedHelper.getStagingDir(cluster, feed, tableStorage, Tag.REPLICATION);
        //stagingDir/dataOutPartitionValue/nominal-time/clusterName/data
        Path stagingPath = new Path(stagingDir + "/*/*/*/*");
        FileSystem fs = getFileSystem(cluster);
        try {
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        final InputStream inputStream = getClass().getResourceAsStream("/config/feed/hive-table-feed.xml");
        Feed tableFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(inputStream);
        getStore().publish(EntityType.FEED, tableFeed);

        final Cluster srcCluster = dfsCluster.getCluster();
        final CatalogStorage sourceStorage = (CatalogStorage) FeedHelper.createStorage(srcCluster, tableFeed);
        String sourceStagingDir = FeedHelper.getStagingDir(srcCluster, tableFeed, sourceStorage, Tag.REPLICATION);

        sourceStagingPath1 = new Path(sourceStagingDir + "/ds=2012092400/" + System.currentTimeMillis());
        sourceStagingPath2 = new Path(sourceStagingDir + "/ds=2012092500/" + System.currentTimeMillis());

        final Cluster targetCluster = targetDfsCluster.getCluster();
        final CatalogStorage targetStorage = (CatalogStorage) FeedHelper.createStorage(targetCluster, tableFeed);
        String targetStagingDir = FeedHelper.getStagingDir(targetCluster, tableFeed, targetStorage, Tag.REPLICATION);

        targetStagingPath1 = new Path(targetStagingDir + "/ds=2012092400/" + System.currentTimeMillis());
        targetStagingPath2 = new Path(targetStagingDir + "/ds=2012092500/" + System.currentTimeMillis());
    }
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.CatalogStorage

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
  • org.apache.falcon.converter.OozieProcessMapperTest
  • org.apache.falcon.converter.OozieProcessWorkflowBuilderTest
  • org.apache.falcon.entity.parser.FeedEntityParser
  • org.apache.falcon.latedata.LateDataHandler
  • org.apache.falcon.metadata.InstanceRelationshipGraphBuilder
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.