Package org.apache.ivory.entity.v0.process

Examples of org.apache.ivory.entity.v0.process.LateInput


        retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
        return retentionApp;
    }

    private ACTION getRetentionWorkflowAction(Cluster cluster, Path wfPath, String wfName) throws IvoryException {
        Feed feed = getEntity();
        ACTION retentionAction = new ACTION();
        WORKFLOW retentionWorkflow = new WORKFLOW();
        try {
            //
            WORKFLOWAPP retWfApp = createRetentionWorkflow(cluster);
            retWfApp.setName(wfName);
            marshal(cluster, retWfApp, wfPath);
            retentionWorkflow.setAppPath(getStoragePath(wfPath.toString()));

            Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);

            org.apache.ivory.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
            String feedPathMask = getLocationURI(cluster, feed,LocationType.DATA);
      String metaPathMask = getLocationURI(cluster, feed, LocationType.META);
            String statsPathMask = getLocationURI(cluster, feed, LocationType.STATS);
            String tmpPathMask = getLocationURI(cluster, feed, LocationType.TMP);

            StringBuilder feedBasePaths = new StringBuilder(feedPathMask);
            if(metaPathMask!=null){
              feedBasePaths.append(FEED_PATH_SEP).append(metaPathMask);
            }
            if(statsPathMask!=null){
              feedBasePaths.append(FEED_PATH_SEP).append(statsPathMask);
            }
            if(tmpPathMask!=null){
              feedBasePaths.append(FEED_PATH_SEP).append(tmpPathMask);
            }

            props.put("feedDataPath", feedBasePaths.toString().replaceAll("\\$\\{", "\\?\\{"));
            props.put("timeZone", feed.getTimezone().getID());
            props.put("frequency", feed.getFrequency().getTimeUnit().name());
            props.put("limit", feedCluster.getRetention().getLimit().toString());
            props.put(ARG.operation.getPropName(), EntityOps.DELETE.name());
            props.put(ARG.feedNames.getPropName(), feed.getName());
            props.put(ARG.feedInstancePaths.getPropName(), "IGNORE");

            retentionWorkflow.setConfiguration(getCoordConfig(props));
            retentionAction.setWorkflow(retentionWorkflow);
            return retentionAction;
View Full Code Here


            // start time >= end time
            return null;

        Cluster cluster = configStore.get(EntityType.CLUSTER, feedCluster.getName());
        Path bundlePath = new Path(ClusterHelper.getLocation(cluster, "staging"), EntityUtil.getStagingPath(feed));
        Feed feedClone = (Feed) feed.clone();
        EntityUtil.setStartDate(feedClone, clusterName, startDate);

        AbstractOozieEntityMapper<Feed> mapper = new OozieFeedMapper(feedClone);
        if(!mapper.map(cluster, bundlePath)){
            return null;
View Full Code Here

            throw new IvoryException("Unable to create parent/retention workflow", e);
        }
    }

    private List<COORDINATORAPP> getReplicationCoordinators(Cluster targetCluster, Path bundlePath) throws IvoryException {
        Feed feed = getEntity();
        List<COORDINATORAPP> replicationCoords = new ArrayList<COORDINATORAPP>();
       
        if (FeedHelper.getCluster(feed, targetCluster.getName()).getType() == ClusterType.TARGET) {
            String coordName = EntityUtil.getWorkflowName(Tag.REPLICATION, feed).toString();
            Path basePath = getCoordPath(bundlePath, coordName);
            createReplicatonWorkflow(targetCluster, basePath, coordName);
           
            for (org.apache.ivory.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
                if (feedCluster.getType() == ClusterType.SOURCE) {
                    COORDINATORAPP coord = createAndGetCoord(feed,
                            (Cluster) ConfigurationStore.get().get(EntityType.CLUSTER, feedCluster.getName()), targetCluster,
                            bundlePath);
          if (coord != null) {
View Full Code Here

    private ACTION getReplicationWorkflowAction(Cluster srcCluster, Cluster trgCluster, Path wfPath, String wfName) throws IvoryException {
        ACTION replicationAction = new ACTION();
        WORKFLOW replicationWF = new WORKFLOW();
        try {
            replicationWF.setAppPath(getStoragePath(wfPath.toString()));
            Feed feed = getEntity();

            String srcPart = FeedHelper.normalizePartitionExpression(FeedHelper.getCluster(feed, srcCluster.getName()).getPartition());
            srcPart = FeedHelper.evaluateClusterExp(srcCluster, srcPart);
            String targetPart = FeedHelper.normalizePartitionExpression(FeedHelper.getCluster(feed, trgCluster.getName()).getPartition());
            targetPart = FeedHelper.evaluateClusterExp(trgCluster, targetPart);
           
            StringBuilder pathsWithPartitions = new StringBuilder();
            pathsWithPartitions.append("${coord:dataIn('input')}/").append(FeedHelper.normalizePartitionExpression(srcPart, targetPart));

            Map<String, String> props = createCoordDefaultConfiguration(trgCluster, wfPath, wfName);
            props.put("srcClusterName", srcCluster.getName());
            props.put("srcClusterColo", srcCluster.getColo());
            props.put(ARG.feedNames.getPropName(), feed.getName());
            props.put(ARG.feedInstancePaths.getPropName(), pathsWithPartitions.toString());
            String parts = pathsWithPartitions.toString().replaceAll("//+", "/");
            parts = StringUtils.stripEnd(parts, "/");
            props.put("sourceRelativePaths", parts);
            props.put("distcpSourcePaths", "${coord:dataIn('input')}");
            props.put("distcpTargetPaths", "${coord:dataOut('output')}");
            props.put("ivoryInPaths", pathsWithPartitions.toString());
            props.put("ivoryInputFeeds", feed.getName());
            replicationWF.setConfiguration(getCoordConfig(props));
            replicationAction.setWorkflow(replicationWF);
        } catch (Exception e) {
            throw new IvoryException("Unable to create replication workflow", e);
        }
View Full Code Here

        return getWorkflowTemplate(RETENTION_WF_TEMPLATE);
    }

    @Override
    protected Map<String, String> getEntityProperties() {
        Feed feed = getEntity();
        Map<String, String> props = new HashMap<String, String>();
        if (feed.getProperties() != null) {
            for (Property prop : feed.getProperties().getProperties())
                props.put(prop.getName(), prop.getValue());
        }
        return props;
    }
View Full Code Here

        Assert.assertEquals(process.getOutputs().getOutputs().get(0).getInstance(), "today(0,0)");

        Assert.assertEquals(process.getProperties().getProperties().get(0).getName(), "name1");
        Assert.assertEquals(process.getProperties().getProperties().get(0).getValue(), "value1");

        Cluster processCluster = process.getClusters().getClusters().get(0);
        Assert.assertEquals(SchemaHelper.formatDateUTC(processCluster.getValidity().getStart()), "2011-11-02T00:00Z");
        Assert.assertEquals(SchemaHelper.formatDateUTC(processCluster.getValidity().getEnd()), "2011-12-30T00:00Z");
        Assert.assertEquals(process.getTimezone().getID(), "UTC");

        Assert.assertEquals(process.getWorkflow().getEngine().name().toLowerCase(), "oozie");
        Assert.assertEquals(process.getWorkflow().getPath(), "/path/to/workflow");
View Full Code Here

    }

    private Feed addInput(Process process, String feed, Cluster cluster) {
        if (process.getInputs() == null) process.setInputs(new Inputs());
        Inputs inputs = process.getInputs();
        Input input = new Input();
        input.setFeed(feed);
        inputs.getInputs().add(input);
        Feed f1 = new Feed();
        f1.setName(feed);
        Clusters clusters = new Clusters();
        f1.setClusters(clusters);
View Full Code Here

    }

    private void attachInput(Process process, Feed feed) {
        if (process.getInputs() == null) process.setInputs(new Inputs());
        Inputs inputs = process.getInputs();
        Input input = new Input();
        input.setFeed(feed.getName());
        inputs.getInputs().add(input);
    }
View Full Code Here

        Assert.assertTrue(entities.contains(f3));
        Assert.assertTrue(entities.contains(f4));
    }

    private Feed addInput(Process process, String feed, Cluster cluster) {
        if (process.getInputs() == null) process.setInputs(new Inputs());
        Inputs inputs = process.getInputs();
        Input input = new Input();
        input.setFeed(feed);
        inputs.getInputs().add(input);
        Feed f1 = new Feed();
        f1.setName(feed);
        Clusters clusters = new Clusters();
        f1.setClusters(clusters);
        org.apache.ivory.entity.v0.feed.Cluster feedCluster =
View Full Code Here

        clusters.getClusters().add(feedCluster);
        return f1;
    }

    private void attachInput(Process process, Feed feed) {
        if (process.getInputs() == null) process.setInputs(new Inputs());
        Inputs inputs = process.getInputs();
        Input input = new Input();
        input.setFeed(feed.getName());
        inputs.getInputs().add(input);
    }
View Full Code Here

TOP

Related Classes of org.apache.ivory.entity.v0.process.LateInput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.