Package org.apache.falcon.entity.v0.feed

Examples of org.apache.falcon.entity.v0.feed.Cluster

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    }

    @Override
    public void onRemove(Entity entity) throws FalconException {
        if (entity.getEntityType().equals(EntityType.FEED)) {
            Feed feed = (Feed) entity;
            if (StringUtils.isEmpty(feed.getGroups())) {
                return;
            }
            String[] groups = feed.getGroups().split(",");
            for (String group : groups) {
                groupsMapping.get(group).getFeeds().remove(entity.getName());
                if (groupsMapping.get(group).getFeeds().size() == 0) {
                    groupsMapping.remove(group);
                }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
*/
public class TestLateRerunHandler {

    @Test
    public void testFeedCutOff() throws FalconException {
        Feed feed = new Feed();
        LateArrival lateArrival = new LateArrival();
        lateArrival.setCutOff(Frequency.fromString("days(1)"));
        feed.setLateArrival(lateArrival);
        String nm = "2013-10-01T12:00Z";
        Date cutOff = LateRerunHandler.getCutOffTime(feed, nm);
        Assert.assertEquals(EntityUtil.parseDateUTC("2013-10-02T12:00Z"), cutOff);

        lateArrival.setCutOff(Frequency.fromString("days(10000000)"));
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    @Test
    public void testFeedUpdateWithOneDependentProcess() {
        try {
            ConfigurationStore.get().remove(EntityType.FEED, "clicks");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample");
            Feed feed = parser.parseAndValidate(this.getClass()
                    .getResourceAsStream(FEED_XML));
            ConfigurationStore.get().publish(EntityType.FEED, feed);
            storeEntity(EntityType.PROCESS, "sample");

            //Try parsing the same feed xml
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
        try {
            ConfigurationStore.get().remove(EntityType.FEED, "clicks");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample2");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample3");
            Feed feed = parser.parseAndValidate(this.getClass()
                    .getResourceAsStream(FEED_XML));
            ConfigurationStore.get().publish(EntityType.FEED, feed);
            storeEntity(EntityType.PROCESS, "sample");
            storeEntity(EntityType.PROCESS, "sample2");
            storeEntity(EntityType.PROCESS, "sample3");
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
        ConfigurationStore.get().remove(EntityType.PROCESS, "sample");
        ConfigurationStore.get().remove(EntityType.PROCESS, "sample2");
        storeEntity(EntityType.FEED, "impressionFeed");
        storeEntity(EntityType.FEED, "imp-click-join1");
        storeEntity(EntityType.FEED, "imp-click-join2");
        Feed feed = parser.parseAndValidate(this.getClass()
                .getResourceAsStream(FEED_XML));
        ConfigurationStore.get().publish(EntityType.FEED, feed);
        Process process = processParser.parseAndValidate(this.getClass()
                .getResourceAsStream(PROCESS1_XML));
        ConfigurationStore.get().publish(EntityType.PROCESS, process);
View Full Code Here
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
        parser.validate(process);
    }

    @Test
    public void testProcessForTableStorage() throws Exception {
        Feed inFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource("/config/feed/hive-table-feed.xml"));
        getStore().publish(EntityType.FEED, inFeed);

        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource("/config/feed/hive-table-feed-out.xml"));
        getStore().publish(EntityType.FEED, outFeed);

        Process process = parser.parse(
                ProcessEntityParserTest.class.getResourceAsStream("/config/process/process-table.xml"));
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
*/
public class TestWorkflowNameBuilder {

    @Test
    public void getTagTest() {
        Feed feed = new Feed();
        feed.setName("raw-logs");

        WorkflowNameBuilder<Feed> builder = new WorkflowNameBuilder<Feed>(feed);
        Tag tag = builder.getWorkflowTag("FALCON_FEED_RETENTION_raw-logs");
        Assert.assertEquals(tag, Tag.RETENTION);

View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
    }

    @Test
    public void getSuffixesTest() {
        Feed feed = new Feed();
        feed.setName("raw-logs");
        WorkflowNameBuilder<Feed> builder = new WorkflowNameBuilder<Feed>(feed);

        String suffixes = builder
                .getWorkflowSuffixes("FALCON_FEED_REPLICATION_raw-logs_corp-1");
        Assert.assertEquals(suffixes, "_corp-1");
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
        Assert.assertEquals(suffixes, "");
    }

    @Test
    public void workflowNameTest() {
        Feed feed = new Feed();
        feed.setName("raw-logs");

        WorkflowNameBuilder<Feed> builder = new WorkflowNameBuilder<Feed>(feed);
        Assert.assertEquals(builder.getWorkflowName().toString(),
                "FALCON_FEED_raw-logs");
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        return propertiesMap;
    }

    private void addOptionalInputProperties(Properties properties, Input in, String clusterName)
        throws FalconException {
        Feed feed = EntityUtil.getEntity(EntityType.FEED, in.getFeed());
        org.apache.falcon.entity.v0.feed.Cluster cluster = FeedHelper.getCluster(feed, clusterName);
        String inName = in.getName();
        properties.put(inName + ".frequency", String.valueOf(feed.getFrequency().getFrequency()));
        properties.put(inName + ".freq_timeunit", mapToCoordTimeUnit(feed.getFrequency().getTimeUnit()).name());
        properties.put(inName + ".timezone", feed.getTimezone().getID());
        properties.put(inName + ".end_of_duration", Timeunit.NONE.name());
        properties.put(inName + ".initial-instance", SchemaHelper.formatDateUTC(cluster.getValidity().getStart()));
        properties.put(inName + ".done-flag", "notused");

        String locPath = FeedHelper.createStorage(clusterName, feed)
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.v0.feed.Cluster

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.cleanup.ProcessCleanupHandler
  • org.apache.falcon.cluster.util.EmbeddedCluster
  • org.apache.falcon.converter.AbstractTestBase
  • org.apache.falcon.converter.OozieFeedMapper
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.