Package org.apache.falcon.entity.v0.cluster

Examples of org.apache.falcon.entity.v0.cluster.Cluster

150
151
152
153
154
155
156
157
158
159
160
161
    public void onAdd(Entity entity) throws FalconException {
        if (entity.getEntityType() != EntityType.CLUSTER) {
            return;
        }

        Cluster cluster = (Cluster) entity;
        if (!EntityUtil.responsibleFor(cluster.getColo())) {
            return;
        }

        addLibsTo(cluster);
    }
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
    @Override
    public void onChange(Entity oldEntity, Entity newEntity) throws FalconException {
        if (oldEntity.getEntityType() != EntityType.CLUSTER) {
            return;
        }
        Cluster oldCluster = (Cluster) oldEntity;
        Cluster newCluster = (Cluster) newEntity;
        if (!ClusterHelper.getInterface(oldCluster, Interfacetype.WRITE).getEndpoint()
                .equals(ClusterHelper.getInterface(newCluster, Interfacetype.WRITE).getEndpoint())
                || !ClusterHelper.getInterface(oldCluster, Interfacetype.WORKFLOW).getEndpoint()
                .equals(ClusterHelper.getInterface(newCluster, Interfacetype.WORKFLOW).getEndpoint())) {
            addLibsTo(newCluster);
View Full Code Here
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
        action.setRetryMax(props.getProperty("falcon.parentworkflow.retry.max", "3"));
        action.setRetryInterval(props.getProperty("falcon.parentworkflow.retry.interval.secs", "1"));
    }

    protected Properties createAppProperties(String clusterName, Path bundlePath, String user) throws FalconException {
        Cluster cluster = EntityUtil.getEntity(EntityType.CLUSTER, clusterName);
        Properties properties = new Properties();
        if (cluster.getProperties() != null) {
            addClusterProperties(properties, cluster.getProperties().getProperties());
        }
        properties.setProperty(OozieWorkflowEngine.NAME_NODE, ClusterHelper.getStorageUrl(cluster));
        properties.setProperty(OozieWorkflowEngine.JOB_TRACKER, ClusterHelper.getMREndPoint(cluster));
        properties.setProperty(OozieClient.BUNDLE_APP_PATH,
                "${" + OozieWorkflowEngine.NAME_NODE + "}" + bundlePath.toString());
        properties.setProperty("colo.name", cluster.getColo());

        properties.setProperty(OozieClient.USER_NAME, user);
        properties.setProperty(OozieClient.USE_SYSTEM_LIBPATH, "true");
        properties.setProperty("falcon.libpath", ClusterHelper.getLocation(cluster, "working") + "/lib");

        if (isTableStorageType(cluster, entity)) {
            propagateHiveCredentials(cluster, properties);
        }

        LOG.info("Cluster: " + cluster.getName() + ", PROPS: " + properties);
        return properties;
    }
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
        }
    }

    private void commitStagingPath(String cluster, String path) throws FalconException {
        path = StringUtils.removeStart(path, "${nameNode}");
        Cluster clusterEntity = ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
        FileSystem fs = HadoopClientFactory.get().createFileSystem(ClusterHelper.getConfiguration(clusterEntity));
        try {
            fs.create(new Path(path, EntityUtil.SUCCEEDED_FILE_NAME)).close();
        } catch (IOException e) {
            throw new FalconException(e);
View Full Code Here
862
863
864
865
866
867
868
869
870
871
872
        if (!entityUpdated && !wfUpdated) {
            LOG.debug("Nothing to update for cluster " + cluster);
            return null;
        }

        Cluster clusterEntity = ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
        Path stagingPath = EntityUtil.getLastCommittedStagingPath(clusterEntity, oldEntity);
        if (stagingPath != null) {  //update if entity is scheduled
            BundleJob bundle = findBundleForStagingPath(cluster, oldEntity, stagingPath);
            bundle = getBundleInfo(cluster, bundle.getId());
            LOG.info("Updating entity through Workflow Engine" + newEntity.toShortString());
View Full Code Here
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
        Job.Status oldBundleStatus = oldBundle.getStatus();
        //Suspend coords as bundle suspend doesn't suspend coords synchronously
        suspendCoords(cluster, oldBundle);

        Cluster clusterEntity = ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
        Path stagingPath = EntityUtil.getLatestStagingPath(clusterEntity, oldEntity);
        //find last scheduled bundle
        BundleJob latestBundle = findBundleForStagingPath(cluster, oldEntity, stagingPath);
        Date effectiveTime;
        if (oldBundle.getAppPath().endsWith(stagingPath.toUri().getPath()) || latestBundle == null || !alreadyCreated) {
View Full Code Here
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
        configStore.publish(EntityType.FEED, join2Feed);
        outputFeeds.add(join2Feed);
    }

    public static Cluster buildCluster(String name, String colo, String tags) {
        Cluster cluster = new Cluster();
        cluster.setName(name);
        cluster.setColo(colo);
        cluster.setTags(tags);

        Interfaces interfaces = new Interfaces();
        cluster.setInterfaces(interfaces);

        Interface storage = new Interface();
        storage.setEndpoint("jail://global:00");
        storage.setType(Interfacetype.WRITE);
        cluster.getInterfaces().getInterfaces().add(storage);

        return cluster;
    }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
            }

            Set<String> clusters = EntityUtil.getClustersDefined(entity);
            Set<String> colos = new HashSet<String>();
            for (String cluster : clusters) {
                Cluster clusterEntity = EntityUtil.getEntity(EntityType.CLUSTER, cluster);
                colos.add(clusterEntity.getColo());
            }
            return colos;
        } catch (FalconException e) {
            throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
        }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
        Map<String, String> overlay = sourceContext.getUniqueOverlay();
        String sourceFilePath = TestContext.overlayParametersOverTemplate("/table/primary-cluster.xml", overlay);
        sourceContext.setCluster(sourceFilePath);

        final Cluster sourceCluster = sourceContext.getCluster().getCluster();
        String sourceStorageUrl = ClusterHelper.getStorageUrl(sourceCluster);

        // copyTestDataToHDFS
        final String sourcePath = sourceStorageUrl + SOURCE_LOCATION + PARTITION_VALUE;
        FSUtils.copyResourceToHDFS("/apps/data/data.txt", "data.txt", sourcePath);

        String targetFilePath = TestContext.overlayParametersOverTemplate("/table/bcp-cluster.xml", overlay);
        targetContext.setCluster(targetFilePath);

        final Cluster targetCluster = targetContext.getCluster().getCluster();
        copyLibsToHDFS(targetCluster);

        String file = TestContext.overlayParametersOverTemplate("/table/target-cluster-alpha.xml", overlay);
        targetAlphaContext.setCluster(file);
        copyLibsToHDFS(targetAlphaContext.getCluster().getCluster());
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
    }

    @Test (enabled = false)
    public void testFSReplicationComplex() throws Exception {
        // copyTestDataToHDFS
        Cluster sourceCluster = sourceContext.getCluster().getCluster();
        FileSystem sourceFS = FileSystem.get(ClusterHelper.getConfiguration(sourceCluster));
        String sourceStorageUrl = ClusterHelper.getStorageUrl(sourceCluster);
        final String partitionValue = "2012-10-01-12";
        String sourceLocation = sourceStorageUrl + SOURCE_LOCATION + partitionValue + "/" + sourceCluster.getColo();
        FSUtils.copyResourceToHDFS("/apps/data/data.txt", "data.txt", sourceLocation);
        Path sourcePath = new Path(sourceLocation);
        Assert.assertTrue(sourceFS.exists(sourcePath));

        final Map<String, String> overlay = sourceContext.getUniqueOverlay();
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.v0.cluster.Cluster

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.cleanup.ProcessCleanupHandler
  • org.apache.falcon.cluster.util.EmbeddedCluster
  • org.apache.falcon.converter.AbstractTestBase
  • org.apache.falcon.converter.OozieFeedMapper
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.