Package org.apache.ivory.oozie.coordinator

Examples of org.apache.ivory.oozie.coordinator.COORDINATORAPP


    }

    @Override
    protected List<COORDINATORAPP> getCoordinators(Cluster cluster, Path bundlePath) throws IvoryException {
        List<COORDINATORAPP> coords = new ArrayList<COORDINATORAPP>();
        COORDINATORAPP retentionCoord = getRetentionCoordinator(cluster, bundlePath);
        if (retentionCoord != null) {
            coords.add(retentionCoord);
        }
        List<COORDINATORAPP> replicationCoords = getReplicationCoordinators(cluster, bundlePath);
        coords.addAll(replicationCoords);
View Full Code Here


        if (feedCluster.getValidity().getEnd().before(new Date())) {
            LOG.warn("Feed Retention is not applicable as Feed's end time for cluster " + cluster.getName() + " is not in the future");
            return null;
        }
        COORDINATORAPP retentionApp = new COORDINATORAPP();
        String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
        retentionApp.setName(coordName);
        retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
        retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
        retentionApp.setTimezone(feed.getTimezone().getID());
        TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
        if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
            retentionApp.setFrequency("${coord:hours(6)}");
        } else {
            retentionApp.setFrequency("${coord:days(1)}");
        }

        Path wfPath = getCoordPath(bundlePath, coordName);
        retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
        return retentionApp;
    }
View Full Code Here

            Path basePath = getCoordPath(bundlePath, coordName);
            createReplicatonWorkflow(targetCluster, basePath, coordName);
           
            for (org.apache.ivory.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
                if (feedCluster.getType() == ClusterType.SOURCE) {
                    COORDINATORAPP coord = createAndGetCoord(feed,
                            (Cluster) ConfigurationStore.get().get(EntityType.CLUSTER, feedCluster.getName()), targetCluster,
                            bundlePath);
          if (coord != null) {
            replicationCoords.add(coord);
          }
View Full Code Here

        return replicationCoords;
    }

    private COORDINATORAPP createAndGetCoord(Feed feed, Cluster srcCluster, Cluster trgCluster, Path bundlePath)
            throws IvoryException {
        COORDINATORAPP replicationCoord;
        String coordName;
        try {
            replicationCoord = getCoordinatorTemplate(REPLICATION_COORD_TEMPLATE);
            coordName = EntityUtil.getWorkflowName(Tag.REPLICATION, Arrays.asList(srcCluster.getName()), feed).toString();
            replicationCoord.setName(coordName);
            replicationCoord.setFrequency("${coord:" + feed.getFrequency().toString() + "}");

            long frequency_ms = ExpressionHelper.get().
                    evaluate(feed.getFrequency().toString(), Long.class);
            long timeout_ms = frequency_ms * 6;           
            Map<String,String> props = getEntityProperties();
            String timeout = props.get(TIMEOUT);
            if(timeout!=null){
              try{
                timeout_ms= ExpressionHelper.get().
                    evaluate(timeout, Long.class);
              }catch (Exception ignore) {
                LOG.error("Unable to evaluate timeout:", ignore);
              }
            }
            if (timeout_ms < THIRTY_MINUTES) timeout_ms = THIRTY_MINUTES;
           
            String parallelProp = props.get(PARALLEL);
      int parallel = 1;
      if (parallelProp != null) {
        try {
          parallel = Integer.parseInt(parallelProp);
        } catch (NumberFormatException ignore) {
          LOG.error("Unable to parse parallel:", ignore);
        }
      }
     
            replicationCoord.getControls().setTimeout(String.valueOf(timeout_ms / (1000 * 60)));
            replicationCoord.getControls().setThrottle(String.valueOf(timeout_ms / frequency_ms * 2));
            replicationCoord.getControls().setConcurrency(String.valueOf(parallel));
           
      Frequency replicationDelay = FeedHelper.getCluster(feed,
          srcCluster.getName()).getDelay();
      long delay_ms=0;
      if (replicationDelay != null) {
        delay_ms = ExpressionHelper.get().evaluate(
            replicationDelay.toString(), Long.class);
        long delay_mins = -1 * delay_ms / (1000 * 60);
        String elExp = "${now(0," + delay_mins + ")}";
        replicationCoord.getInputEvents().getDataIn().get(0)
            .getInstance().set(0, elExp);
        replicationCoord.getOutputEvents().getDataOut().get(0)
            .setInstance(elExp);
      }

            Date srcStartDate = FeedHelper.getCluster(feed, srcCluster.getName()).getValidity().getStart();
            srcStartDate=new Date(srcStartDate.getTime()+delay_ms);
            Date srcEndDate = FeedHelper.getCluster(feed, srcCluster.getName()).getValidity().getEnd();
            Date trgStartDate = FeedHelper.getCluster(feed, trgCluster.getName()).getValidity().getStart();
            trgStartDate=new Date(trgStartDate.getTime()+delay_ms);
            Date trgEndDate = FeedHelper.getCluster(feed, trgCluster.getName()).getValidity().getEnd();
      if (srcStartDate.after(trgEndDate)
          || trgStartDate.after(srcEndDate)) {
        LOG.warn("Not creating replication coordinator, as the source cluster:"
            + srcCluster.getName()
            + " and target cluster: "
            + trgCluster.getName()
            + " do not have overlapping dates");
        return null;
      }
            replicationCoord.setStart(srcStartDate.after(trgStartDate) ? SchemaHelper.formatDateUTC(srcStartDate) : SchemaHelper
                    .formatDateUTC(trgStartDate));
            replicationCoord.setEnd(srcEndDate.before(trgEndDate) ? SchemaHelper.formatDateUTC(srcEndDate) : SchemaHelper
                    .formatDateUTC(trgEndDate));
            replicationCoord.setTimezone(feed.getTimezone().getID());
            SYNCDATASET inputDataset = (SYNCDATASET) replicationCoord.getDatasets().getDatasetOrAsyncDataset().get(0);
            SYNCDATASET outputDataset = (SYNCDATASET) replicationCoord.getDatasets().getDatasetOrAsyncDataset().get(1);

      inputDataset.setUriTemplate(new Path(ClusterHelper
          .getStorageUrl(srcCluster), FeedHelper.getLocation(feed,
          LocationType.DATA,srcCluster.getName()).getPath()).toString());
      outputDataset.setUriTemplate(getStoragePath(FeedHelper.getLocation(
          feed, LocationType.DATA, trgCluster.getName()).getPath()));
            setDatasetValues(inputDataset, feed, srcCluster);
            setDatasetValues(outputDataset, feed, srcCluster);
            if (feed.getAvailabilityFlag() == null) {
                inputDataset.setDoneFlag("");
            } else {
                inputDataset.setDoneFlag(feed.getAvailabilityFlag());
            }

        } catch (IvoryException e) {
            throw new IvoryException("Cannot unmarshall replication coordinator template", e);
        }

        Path wfPath = getCoordPath(bundlePath, coordName);
        replicationCoord.setAction(getReplicationWorkflowAction(srcCluster, trgCluster, wfPath, coordName));
        return replicationCoord;
    }
View Full Code Here

  @Test
  public void testReplicationCoords() throws IvoryException {
    OozieFeedMapper feedMapper = new OozieFeedMapper(feed);
    List<COORDINATORAPP> coords = feedMapper.getCoordinators(trgCluster,
        new Path("/projects/ivory/"));
    COORDINATORAPP coord = coords.get(0);
    Assert.assertEquals("2010-01-01T00:40Z", coord.getStart());   
    Assert.assertEquals("${nameNode}/projects/ivory/REPLICATION", coord
        .getAction().getWorkflow().getAppPath());
    Assert.assertEquals("IVORY_FEED_REPLICATION_" + feed.getName() + "_"
        + srcCluster.getName(), coord.getName());
    Assert.assertEquals("${coord:minutes(20)}", coord.getFrequency());
    SYNCDATASET inputDataset = (SYNCDATASET) coord.getDatasets()
        .getDatasetOrAsyncDataset().get(0);
    SYNCDATASET outputDataset = (SYNCDATASET) coord.getDatasets()
        .getDatasetOrAsyncDataset().get(1);

    Assert.assertEquals("${coord:minutes(20)}", inputDataset.getFrequency());
    Assert.assertEquals("input-dataset", inputDataset.getName());
    Assert.assertEquals(
        ClusterHelper.getStorageUrl(srcCluster)
            + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
        inputDataset.getUriTemplate());

    Assert.assertEquals("${coord:minutes(20)}",
        outputDataset.getFrequency());
    Assert.assertEquals("output-dataset", outputDataset.getName());
    Assert.assertEquals(
        "${nameNode}"
            + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
        outputDataset.getUriTemplate());
   
    String inEventName =coord.getInputEvents().getDataIn().get(0).getName();
    String inEventDataset =coord.getInputEvents().getDataIn().get(0).getDataset();
    String inEventInstance = coord.getInputEvents().getDataIn().get(0).getInstance().get(0);
    Assert.assertEquals("input", inEventName);
    Assert.assertEquals("input-dataset", inEventDataset);
    Assert.assertEquals("${now(0,-40)}", inEventInstance);

    String outEventInstance = coord.getOutputEvents().getDataOut().get(0).getInstance();
    Assert.assertEquals("${now(0,-40)}", outEventInstance);
   
        for(Property prop:coord.getAction().getWorkflow().getConfiguration().getProperty()){
          if(prop.getName().equals("mapred.job.priority")){
            assertEquals(prop.getValue(), "NORMAL");
            break;
          }
        }
View Full Code Here

        assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT,process).toString(), bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");
       
        COORDINATORAPP coord = getCoordinator(fs, new Path(coordPath));
        testDefCoordMap(process, coord);
        assertEquals(coord.getControls().getThrottle(), "12");
        assertEquals(coord.getControls().getTimeout(), "360");

        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        WORKFLOWAPP parentWorkflow = getParentWorkflow(fs, new Path(wfPath));
        testParentWorkflow(process,parentWorkflow);
    }
View Full Code Here

        assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT,process).toString(), bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");

        COORDINATORAPP coord = getCoordinator(fs, new Path(coordPath));
        testDefCoordMap(process, coord);
        assertEquals(coord.getControls().getThrottle(), "30");
        assertEquals(coord.getControls().getTimeout(), "15");

        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        WORKFLOWAPP parentWorkflow = getParentWorkflow(fs, new Path(wfPath));
        testParentWorkflow(process,parentWorkflow);
    }
View Full Code Here

    public COORDINATORAPP createDefaultCoordinator(Cluster cluster, Path bundlePath) throws IvoryException {
        Process process = getEntity();
        if (process == null)
            return null;

        COORDINATORAPP coord = new COORDINATORAPP();
        String coordName = EntityUtil.getWorkflowName(Tag.DEFAULT,process).toString();
        Path coordPath = getCoordPath(bundlePath, coordName);

        // coord attributes
        coord.setName(coordName);
        org.apache.ivory.entity.v0.process.Cluster processCluster = ProcessHelper.getCluster(process, cluster.getName());
        coord.setStart(SchemaHelper.formatDateUTC(processCluster.getValidity().getStart()));
        coord.setEnd(SchemaHelper.formatDateUTC(processCluster.getValidity().getEnd()));
        coord.setTimezone(process.getTimezone().getID());
        coord.setFrequency("${coord:" + process.getFrequency().toString() + "}");

        // controls
        CONTROLS controls = new CONTROLS();
        controls.setConcurrency(String.valueOf(process.getParallel()));
        controls.setExecution(process.getOrder().name());

        Frequency timeout = process.getTimeout();
        long frequency_ms = ExpressionHelper.get().
                evaluate(process.getFrequency().toString(), Long.class);
        long timeout_ms;
        if (timeout != null) {
            timeout_ms = ExpressionHelper.get().
                    evaluate(process.getTimeout().toString(), Long.class);
        } else {
            timeout_ms = frequency_ms * 6;
            if (timeout_ms < THIRTY_MINUTES) timeout_ms = THIRTY_MINUTES;
        }
        controls.setTimeout(String.valueOf(timeout_ms / (1000 * 60)));
        if (timeout_ms / frequency_ms * 2 > 0) {
            controls.setThrottle(String.valueOf(timeout_ms / frequency_ms * 2));
        }
        coord.setControls(controls);

        // Configuration
        Map<String, String> props = createCoordDefaultConfiguration(cluster, coordPath, coordName);

        List<String> inputFeeds = new ArrayList<String>();
        List<String> inputPaths = new ArrayList<String>();
        // inputs
        if (process.getInputs() != null) {           
            for (Input input : process.getInputs().getInputs()) {
                if(!input.isOptional()) {
                    if (coord.getDatasets() == null)
                        coord.setDatasets(new DATASETS());
                    if (coord.getInputEvents() == null)
                        coord.setInputEvents(new INPUTEVENTS());

                    SYNCDATASET syncdataset = createDataSet(input.getFeed(), cluster, input.getName(), LocationType.DATA);
                    coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);
   
                    DATAIN datain = createDataIn(input);
                    coord.getInputEvents().getDataIn().add(datain);
                }

                String inputExpr = getELExpression("dataIn('" + input.getName() + "', '" + input.getPartition() + "')");
                props.put(input.getName(), inputExpr);
                inputFeeds.add(input.getName());
                inputPaths.add(inputExpr);

            }
        }
        props.put("ivoryInPaths", join(inputPaths.iterator(), '#'));
        props.put("ivoryInputFeeds", join(inputFeeds.iterator(), '#'));

        // outputs
        List<String> outputFeeds = new ArrayList<String>();
        List<String> outputPaths = new ArrayList<String>();
        if (process.getOutputs() != null) {
            if (coord.getDatasets() == null)
                coord.setDatasets(new DATASETS());
            if (coord.getOutputEvents() == null)
                coord.setOutputEvents(new OUTPUTEVENTS());
           
            for (Output output : process.getOutputs().getOutputs()) {
                SYNCDATASET syncdataset = createDataSet(output.getFeed(), cluster, output.getName(),LocationType.DATA);
                coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);

                DATAOUT dataout = createDataOut(output);
                coord.getOutputEvents().getDataOut().add(dataout);

                String outputExpr = "${coord:dataOut('" + output.getName() + "')}";
                props.put(output.getName(), outputExpr);
                outputFeeds.add(output.getName());
                outputPaths.add(outputExpr);
               
        // stats and meta paths
        createOutputEvent(output.getFeed(),output.getName(), cluster, "stats",
            LocationType.STATS, coord, props, output.getInstance());
        createOutputEvent(output.getFeed(),output.getName(), cluster, "meta",
            LocationType.META, coord, props,output.getInstance());
        createOutputEvent(output.getFeed(),output.getName(), cluster, "tmp",
            LocationType.TMP, coord, props,output.getInstance());

            }
        }
        // Output feed name and path for parent workflow
        props.put(ARG.feedNames.getPropName(), join(outputFeeds.iterator(), ','));
        props.put(ARG.feedInstancePaths.getPropName(), join(outputPaths.iterator(), ','));

        // create parent wf
        createWorkflow(cluster, DEFAULT_WF_TEMPLATE, coordName, coordPath);

        WORKFLOW wf = new WORKFLOW();
        wf.setAppPath(getStoragePath(coordPath.toString()));
        wf.setConfiguration(getCoordConfig(props));

        // set coord action to parent wf
        org.apache.ivory.oozie.coordinator.ACTION action = new org.apache.ivory.oozie.coordinator.ACTION();
        action.setWorkflow(wf);
        coord.setAction(action);

        return coord;
    }
View Full Code Here

TOP

Related Classes of org.apache.ivory.oozie.coordinator.COORDINATORAPP

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.