Package org.apache.falcon.entity.v0.process

Examples of org.apache.falcon.entity.v0.process.Clusters


        STORE.publish(EntityType.FEED, feed1);
        STORE.publish(EntityType.FEED, feed2);
        STORE.publish(EntityType.FEED, feed3);

        Process process1 = (Process) EntityType.PROCESS.getUnmarshaller()
                .unmarshal(this.getClass().getResource(PROCESS1_XML));
        STORE.publish(EntityType.PROCESS, process1);
        Process process2 = (Process) EntityType.PROCESS.getUnmarshaller()
                .unmarshal(this.getClass().getResource(PROCESS2_XML));
        STORE.publish(EntityType.PROCESS, process2);
    }
View Full Code Here


     * @param bundlePath - bundle path
     * @return COORDINATORAPP
     * @throws FalconException on Error
     */
    public COORDINATORAPP createDefaultCoordinator(Cluster cluster, Path bundlePath) throws FalconException {
        Process process = getEntity();
        if (process == null) {
            return null;
        }

        COORDINATORAPP coord = new COORDINATORAPP();
        String coordName = EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString();
        Path coordPath = getCoordPath(bundlePath, coordName);

        // coord attributes
        initializeCoordAttributes(cluster, process, coord, coordName);

        CONTROLS controls = initializeControls(process); // controls
        coord.setControls(controls);

        // Configuration
        Map<String, String> props = createCoordDefaultConfiguration(cluster, coordPath, coordName);

        initializeInputPaths(cluster, process, coord, props); // inputs
        initializeOutputPaths(cluster, process, coord, props)// outputs

        Workflow processWorkflow = process.getWorkflow();
        props.put("userWorkflowEngine", processWorkflow.getEngine().value());

        // create parent wf
        createWorkflow(cluster, process, processWorkflow, coordName, coordPath);

View Full Code Here

        return expr;
    }

    @Override
    protected Map<String, String> getEntityProperties() {
        Process process = getEntity();
        Map<String, String> props = new HashMap<String, String>();
        if (process.getProperties() != null) {
            for (Property prop : process.getProperties().getProperties()) {
                props.put(prop.getName(), prop.getValue());
            }
        }
        return props;
    }
View Full Code Here

        ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(hdfsUrl);
        ClusterHelper.getInterface(cluster, Interfacetype.REGISTRY).setEndpoint("thrift://localhost:49083");
        fs = new Path(hdfsUrl).getFileSystem(new Configuration());
        fs.create(new Path(ClusterHelper.getLocation(cluster, "working"), "libext/PROCESS/ext.jar")).close();

        Process process = store.get(EntityType.PROCESS, "clicksummary");
        Path wfpath = new Path(process.getWorkflow().getPath());
        assert new Path(hdfsUrl).getFileSystem(new Configuration()).mkdirs(wfpath);
    }
View Full Code Here

    public void testBundle() throws Exception {
        String path = StartupProperties.get().getProperty("system.lib.location");
        if (!new File(path).exists()) {
            Assert.assertTrue(new File(path).mkdirs());
        }
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "clicksummary");

        WORKFLOWAPP parentWorkflow = initializeProcessMapper(process, "12", "360");
        testParentWorkflow(process, parentWorkflow);
    }
View Full Code Here

        testParentWorkflow(process, parentWorkflow);
    }

    @Test
    public void testBundle1() throws Exception {
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "clicksummary");
        process.setFrequency(Frequency.fromString("minutes(1)"));
        process.setTimeout(Frequency.fromString("minutes(15)"));

        WORKFLOWAPP parentWorkflow = initializeProcessMapper(process, "30", "15");
        testParentWorkflow(process, parentWorkflow);
    }
View Full Code Here

        testParentWorkflow(process, parentWorkflow);
    }

    @Test
    public void testPigProcessMapper() throws Exception {
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "pig-process");
        Assert.assertEquals("pig", process.getWorkflow().getEngine().value());

        WORKFLOWAPP parentWorkflow = initializeProcessMapper(process, "12", "360");
        testParentWorkflow(process, parentWorkflow);

        List<Object> decisionOrForkOrJoin = parentWorkflow.getDecisionOrForkOrJoin();
View Full Code Here

        resource = this.getClass().getResource("/config/feed/hive-table-feed-out.xml");
        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.FEED, outFeed);

        resource = this.getClass().getResource("/config/process/hive-process.xml");
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.PROCESS, process);

        Cluster cluster = ConfigurationStore.get().get(EntityType.CLUSTER, "corp");
        OozieProcessMapper mapper = new OozieProcessMapper(process);
        Path bundlePath = new Path("/tmp/seetharam", EntityUtil.getStagingPath(process));
View Full Code Here

        resource = this.getClass().getResource("/config/feed/hive-table-feed-out.xml");
        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.FEED, outFeed);

        resource = this.getClass().getResource("/config/process/pig-process-table.xml");
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.PROCESS, process);

        Cluster cluster = ConfigurationStore.get().get(EntityType.CLUSTER, "corp");
        OozieProcessMapper mapper = new OozieProcessMapper(process);
        Path bundlePath = new Path("/", EntityUtil.getStagingPath(process));
        mapper.map(cluster, bundlePath);
        assertTrue(fs.exists(bundlePath));

        BUNDLEAPP bundle = getBundle(bundlePath);
        assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");

        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }

        // verify table props
        Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process, cluster);
        for (Map.Entry<String, String> entry : props.entrySet()) {
            if (expected.containsKey(entry.getKey())) {
                Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
            }
        }

        // verify the late data params
        Assert.assertEquals(props.get("falconInputFeeds"), process.getInputs().getInputs().get(0).getName());
        Assert.assertEquals(props.get("falconInPaths"), "${coord:dataIn('input')}");
        Assert.assertEquals(props.get("falconInputFeedStorageTypes"), Storage.TYPE.TABLE.name());

        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), process.getOutputs().getOutputs().get(0).getName());
        Assert.assertEquals(props.get("feedInstancePaths"), "${coord:dataOut('output')}");
    }
View Full Code Here

        Map<String, String> props = createCoordDefaultConfiguration(cluster, coordPath, coordName);

        initializeInputPaths(cluster, process, coord, props); // inputs
        initializeOutputPaths(cluster, process, coord, props)// outputs

        Workflow processWorkflow = process.getWorkflow();
        props.put("userWorkflowEngine", processWorkflow.getEngine().value());

        // create parent wf
        createWorkflow(cluster, process, processWorkflow, coordName, coordPath);

        WORKFLOW wf = new WORKFLOW();
View Full Code Here

TOP

Related Classes of org.apache.falcon.entity.v0.process.Clusters

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.