Examples of SYNCDATASET

com.sleepycat.je.sync.SyncDataSet
Defines a sychronized data set as a set of local databases to be synchronized with an external system. An instance of this class is created by calling SyncProcessor.addDataSet.
org.apache.falcon.oozie.coordinator.SYNCDATASET
org.apache.ivory.oozie.coordinator.SYNCDATASET

Examples of org.apache.falcon.oozie.coordinator.SYNCDATASET

        Assert.assertEquals("${nameNode}/projects/falcon/REPLICATION", coord
                .getAction().getWorkflow().getAppPath());
        Assert.assertEquals("FALCON_FEED_REPLICATION_" + feed.getName() + "_"
                + srcCluster.getName(), coord.getName());
        Assert.assertEquals("${coord:minutes(20)}", coord.getFrequency());
        SYNCDATASET inputDataset = (SYNCDATASET) coord.getDatasets()
                .getDatasetOrAsyncDataset().get(0);
        SYNCDATASET outputDataset = (SYNCDATASET) coord.getDatasets()
                .getDatasetOrAsyncDataset().get(1);


        Assert.assertEquals("${coord:minutes(20)}", inputDataset.getFrequency());
        Assert.assertEquals("input-dataset", inputDataset.getName());
        Assert.assertEquals(
                ClusterHelper.getStorageUrl(srcCluster)
                        + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
                inputDataset.getUriTemplate());


        Assert.assertEquals("${coord:minutes(20)}",
                outputDataset.getFrequency());
        Assert.assertEquals("output-dataset", outputDataset.getName());
        Assert.assertEquals(
                "${nameNode}"
                        + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
                        outputDataset.getUriTemplate());
        String inEventName =coord.getInputEvents().getDataIn().get(0).getName();
        String inEventDataset =coord.getInputEvents().getDataIn().get(0).getDataset();
        String inEventInstance = coord.getInputEvents().getDataIn().get(0).getInstance().get(0);
        Assert.assertEquals("input", inEventName);
        Assert.assertEquals("input-dataset", inEventDataset);

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.SYNCDATASET

                }
                if (coord.getInputEvents() == null) {
                    coord.setInputEvents(new INPUTEVENTS());
                }


                SYNCDATASET syncdataset = createDataSet(input.getFeed(), cluster, input.getName(),
                        LocationType.DATA);
                coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);


                DATAIN datain = createDataIn(input);
                coord.getInputEvents().getDataIn().add(datain);

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.SYNCDATASET

        if (coord.getOutputEvents() == null) {
            coord.setOutputEvents(new OUTPUTEVENTS());
        }


        for (Output output : process.getOutputs().getOutputs()) {
            SYNCDATASET syncdataset = createDataSet(output.getFeed(), cluster, output.getName(), LocationType.DATA);
            coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);


            DATAOUT dataout = createDataOut(output);
            coord.getOutputEvents().getDataOut().add(dataout);

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.SYNCDATASET


    //SUSPEND CHECKSTYLE CHECK VisibilityModifierCheck
    private void createOutputEvent(String feed, String name, Cluster cluster,
                                   String type, LocationType locType, COORDINATORAPP coord,
                                   Map<String, String> props, String instance) throws FalconException {
        SYNCDATASET dataset = createDataSet(feed, cluster, name + type,
                locType);
        coord.getDatasets().getDatasetOrAsyncDataset().add(dataset);
        DATAOUT dataout = new DATAOUT();
        if (coord.getOutputEvents() == null) {
            coord.setOutputEvents(new OUTPUTEVENTS());

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.SYNCDATASET


    private SYNCDATASET createDataSet(String feedName, Cluster cluster, String datasetName,
                                      LocationType locationType) throws FalconException {
        Feed feed = EntityUtil.getEntity(EntityType.FEED, feedName);


        SYNCDATASET syncdataset = new SYNCDATASET();
        syncdataset.setName(datasetName);
        String locPath = FeedHelper.getLocation(feed, locationType,
                cluster.getName()).getPath();
        syncdataset.setUriTemplate(new Path(locPath).toUri().getScheme() != null ? locPath : "${nameNode}"
                + locPath);
        syncdataset.setFrequency("${coord:" + feed.getFrequency().toString() + "}");


        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
        syncdataset.setInitialInstance(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
        syncdataset.setTimezone(feed.getTimezone().getID());
        if (feed.getAvailabilityFlag() == null) {
            syncdataset.setDoneFlag("");
        } else {
            syncdataset.setDoneFlag(feed.getAvailabilityFlag());
        }
        return syncdataset;
    }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.SYNCDATASET

                            .formatDateUTC(trgStartDate));
            replicationCoord.setEnd(
                    srcEndDate.before(trgEndDate) ? SchemaHelper.formatDateUTC(srcEndDate) : SchemaHelper
                            .formatDateUTC(trgEndDate));
            replicationCoord.setTimezone(feed.getTimezone().getID());
            SYNCDATASET inputDataset = (SYNCDATASET) replicationCoord.getDatasets().getDatasetOrAsyncDataset().get(0);
            SYNCDATASET outputDataset = (SYNCDATASET) replicationCoord.getDatasets().getDatasetOrAsyncDataset().get(1);


            inputDataset.setUriTemplate(new Path(ClusterHelper.getStorageUrl(srcCluster),
                FeedHelper.getLocation(feed, LocationType.DATA, srcCluster.getName()).getPath()).toString());
            outputDataset.setUriTemplate(getStoragePath(
                FeedHelper.getLocation(feed, LocationType.DATA, trgCluster.getName()).getPath()));
            setDatasetValues(inputDataset, feed, srcCluster);
            setDatasetValues(outputDataset, feed, srcCluster);
            if (feed.getAvailabilityFlag() == null) {
                inputDataset.setDoneFlag("");

View Full Code Here

Examples of org.apache.ivory.oozie.coordinator.SYNCDATASET

            replicationCoord.setStart(srcStartDate.after(trgStartDate) ? SchemaHelper.formatDateUTC(srcStartDate) : SchemaHelper
                    .formatDateUTC(trgStartDate));
            replicationCoord.setEnd(srcEndDate.before(trgEndDate) ? SchemaHelper.formatDateUTC(srcEndDate) : SchemaHelper
                    .formatDateUTC(trgEndDate));
            replicationCoord.setTimezone(feed.getTimezone().getID());
            SYNCDATASET inputDataset = (SYNCDATASET) replicationCoord.getDatasets().getDatasetOrAsyncDataset().get(0);
            SYNCDATASET outputDataset = (SYNCDATASET) replicationCoord.getDatasets().getDatasetOrAsyncDataset().get(1);


      inputDataset.setUriTemplate(new Path(ClusterHelper
          .getStorageUrl(srcCluster), FeedHelper.getLocation(feed,
          LocationType.DATA,srcCluster.getName()).getPath()).toString());
      outputDataset.setUriTemplate(getStoragePath(FeedHelper.getLocation(
          feed, LocationType.DATA, trgCluster.getName()).getPath()));
            setDatasetValues(inputDataset, feed, srcCluster);
            setDatasetValues(outputDataset, feed, srcCluster);
            if (feed.getAvailabilityFlag() == null) {
                inputDataset.setDoneFlag("");

View Full Code Here

Examples of org.apache.ivory.oozie.coordinator.SYNCDATASET

    Assert.assertEquals("${nameNode}/projects/ivory/REPLICATION", coord
        .getAction().getWorkflow().getAppPath());
    Assert.assertEquals("IVORY_FEED_REPLICATION_" + feed.getName() + "_"
        + srcCluster.getName(), coord.getName());
    Assert.assertEquals("${coord:minutes(20)}", coord.getFrequency());
    SYNCDATASET inputDataset = (SYNCDATASET) coord.getDatasets()
        .getDatasetOrAsyncDataset().get(0);
    SYNCDATASET outputDataset = (SYNCDATASET) coord.getDatasets()
        .getDatasetOrAsyncDataset().get(1);


    Assert.assertEquals("${coord:minutes(20)}", inputDataset.getFrequency());
    Assert.assertEquals("input-dataset", inputDataset.getName());
    Assert.assertEquals(
        ClusterHelper.getStorageUrl(srcCluster)
            + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
        inputDataset.getUriTemplate());


    Assert.assertEquals("${coord:minutes(20)}",
        outputDataset.getFrequency());
    Assert.assertEquals("output-dataset", outputDataset.getName());
    Assert.assertEquals(
        "${nameNode}"
            + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
        outputDataset.getUriTemplate());
    
    String inEventName =coord.getInputEvents().getDataIn().get(0).getName();
    String inEventDataset =coord.getInputEvents().getDataIn().get(0).getDataset();
    String inEventInstance = coord.getInputEvents().getDataIn().get(0).getInstance().get(0);
    Assert.assertEquals("input", inEventName);

View Full Code Here

Examples of org.apache.ivory.oozie.coordinator.SYNCDATASET


        assertEquals(6, coord.getDatasets().getDatasetOrAsyncDataset().size());
        
        ConfigurationStore store = ConfigurationStore.get();
        Feed feed = store.get(EntityType.FEED, process.getInputs().getInputs().get(0).getFeed());
        SYNCDATASET ds = (SYNCDATASET) coord.getDatasets().getDatasetOrAsyncDataset().get(0);
        assertEquals(SchemaHelper.formatDateUTC(feed.getClusters().getClusters().get(0).getValidity().getStart()), ds.getInitialInstance());
        assertEquals(feed.getTimezone().getID(), ds.getTimezone());
        assertEquals("${coord:"+feed.getFrequency().toString()+"}", ds.getFrequency());
        assertEquals("", ds.getDoneFlag());
        assertEquals(ds.getUriTemplate(),"${nameNode}" + FeedHelper.getLocation(feed, LocationType.DATA,feed.getClusters().getClusters().get(0).getName()).getPath());   
        for(Property prop:coord.getAction().getWorkflow().getConfiguration().getProperty()){
          if(prop.getName().equals("mapred.job.priority")){
            assertEquals(prop.getValue(), "LOW");
            break;
          }

View Full Code Here

Examples of org.apache.ivory.oozie.coordinator.SYNCDATASET

                    if (coord.getDatasets() == null)
                        coord.setDatasets(new DATASETS());
                    if (coord.getInputEvents() == null)
                        coord.setInputEvents(new INPUTEVENTS());


                    SYNCDATASET syncdataset = createDataSet(input.getFeed(), cluster, input.getName(), LocationType.DATA);
                    coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);
    
                    DATAIN datain = createDataIn(input);
                    coord.getInputEvents().getDataIn().add(datain);
                }


                String inputExpr = getELExpression("dataIn('" + input.getName() + "', '" + input.getPartition() + "')");
                props.put(input.getName(), inputExpr);
                inputFeeds.add(input.getName());
                inputPaths.add(inputExpr);


            }
        }
        props.put("ivoryInPaths", join(inputPaths.iterator(), '#'));
        props.put("ivoryInputFeeds", join(inputFeeds.iterator(), '#'));


        // outputs
        List<String> outputFeeds = new ArrayList<String>();
        List<String> outputPaths = new ArrayList<String>();
        if (process.getOutputs() != null) {
            if (coord.getDatasets() == null)
                coord.setDatasets(new DATASETS());
            if (coord.getOutputEvents() == null)
                coord.setOutputEvents(new OUTPUTEVENTS());
            
            for (Output output : process.getOutputs().getOutputs()) {
                SYNCDATASET syncdataset = createDataSet(output.getFeed(), cluster, output.getName(),LocationType.DATA);
                coord.getDatasets().getDatasetOrAsyncDataset().add(syncdataset);


                DATAOUT dataout = createDataOut(output);
                coord.getOutputEvents().getDataOut().add(dataout);

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.