Package org.apache.falcon.entity.v0.process

Examples of org.apache.falcon.entity.v0.process.Cluster

62
63
64
65
66
67
68
69
70
71
72
73
        Assert.assertEquals(suffixes, "");
    }

    @Test
    public void workflowNameTest() {
        Feed feed = new Feed();
        feed.setName("raw-logs");

        WorkflowNameBuilder<Feed> builder = new WorkflowNameBuilder<Feed>(feed);
        Assert.assertEquals(builder.getWorkflowName().toString(),
                "FALCON_FEED_raw-logs");
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        return propertiesMap;
    }

    private void addOptionalInputProperties(Properties properties, Input in, String clusterName)
        throws FalconException {
        Feed feed = EntityUtil.getEntity(EntityType.FEED, in.getFeed());
        org.apache.falcon.entity.v0.feed.Cluster cluster = FeedHelper.getCluster(feed, clusterName);
        String inName = in.getName();
        properties.put(inName + ".frequency", String.valueOf(feed.getFrequency().getFrequency()));
        properties.put(inName + ".freq_timeunit", mapToCoordTimeUnit(feed.getFrequency().getTimeUnit()).name());
        properties.put(inName + ".timezone", feed.getTimezone().getID());
        properties.put(inName + ".end_of_duration", Timeunit.NONE.name());
        properties.put(inName + ".initial-instance", SchemaHelper.formatDateUTC(cluster.getValidity().getStart()));
        properties.put(inName + ".done-flag", "notused");

        String locPath = FeedHelper.createStorage(clusterName, feed)
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
            cluster.setName(name);
            store.publish(type, cluster);
            break;

        case FEED:
            Feed feed = (Feed) unmarshaller.unmarshal(this.getClass().getResource(resource));
            feed.setName(name);
            store.publish(type, feed);
            break;

        case PROCESS:
            Process process = (Process) unmarshaller.unmarshal(this.getClass().getResource(resource));
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
                .unmarshal(this.getClass().getResource(CLUSTER_XML));
        ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(hdfsUrl);

        STORE.publish(EntityType.CLUSTER, cluster);

        Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource(FEED1_XML));
        Feed feed2 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource(FEED2_XML));
        Feed feed3 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource(FEED3_XML));

        STORE.publish(EntityType.FEED, feed1);
        STORE.publish(EntityType.FEED, feed2);
        STORE.publish(EntityType.FEED, feed3);
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
        List<String> inputFeeds = new ArrayList<String>();
        List<String> inputPaths = new ArrayList<String>();
        List<String> inputFeedStorageTypes = new ArrayList<String>();
        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            if (!input.isOptional()) {
                if (coord.getDatasets() == null) {
                    coord.setDatasets(new DATASETS());
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
        }

        List<String> outputFeeds = new ArrayList<String>();
        List<String> outputPaths = new ArrayList<String>();
        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            SYNCDATASET syncdataset = createDataSet(feed, cluster, storage, output.getName(), LocationType.DATA);
            coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);
View Full Code Here
545
546
547
548
549
550
551
552
553
554
555
    }

    private List<String> getPrepareDeleteOutputPathList(Process process) throws FalconException {
        final List<String> deleteList = new ArrayList<String>();
        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());

            if (FeedHelper.getStorageType(feed) == Storage.TYPE.TABLE) {
                continue; // prepare delete only applies to FileSystem storage
            }
View Full Code Here
560
561
562
563
564
565
566
567
568
569
570
    }

    private void addInputFeedsAsParams(List<String> paramList, Process process, Cluster cluster,
                                       String engineType) throws FalconException {
        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            final String inputName = input.getName();
            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                paramList.add(inputName + "=${" + inputName + "}"); // no prefix for backwards compatibility
View Full Code Here
583
584
585
586
587
588
589
590
591
592
593
    }

    private void addOutputFeedsAsParams(List<String> paramList, Process process,
                                        Cluster cluster) throws FalconException {
        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                final String outputName = output.getName();  // no prefix for backwards compatibility
                paramList.add(outputName + "=${" + outputName + "}");
View Full Code Here
656
657
658
659
660
661
662
663
664
665
666
        if (process.getInputs() == null) {
            return storageType;
        }

        for (Input input : process.getInputs().getInputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, input.getFeed());
            storageType = FeedHelper.getStorageType(feed, cluster);
            if (Storage.TYPE.TABLE == storageType) {
                break;
            }
        }
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.v0.process.Cluster

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.cleanup.ProcessCleanupHandler
  • org.apache.falcon.cluster.util.EmbeddedCluster
  • org.apache.falcon.converter.AbstractTestBase
  • org.apache.falcon.converter.OozieFeedMapper
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.