Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.AbstractJobVertex


     
      final JobID jobId = new JobID();
      final String jobName = "Test Job Sample Name";
      final Configuration cfg = new Configuration();
     
      AbstractJobVertex v1 = new AbstractJobVertex("vertex1");
      AbstractJobVertex v2 = new AbstractJobVertex("vertex2");
      AbstractJobVertex v3 = new AbstractJobVertex("vertex3");
      AbstractJobVertex v4 = new AbstractJobVertex("vertex4");
      AbstractJobVertex v5 = new AbstractJobVertex("vertex5");
     
      v1.setParallelism(5);
      v2.setParallelism(7);
      v3.setParallelism(2);
      v4.setParallelism(11);
      v5.setParallelism(4);
     
      v2.connectNewDataSetAsInput(v1, DistributionPattern.BIPARTITE);
      v4.connectNewDataSetAsInput(v2, DistributionPattern.BIPARTITE);
      v4.connectNewDataSetAsInput(v3, DistributionPattern.BIPARTITE);
      v5.connectNewDataSetAsInput(v4, DistributionPattern.BIPARTITE);
      v5.connectNewDataSetAsInput(v3, DistributionPattern.BIPARTITE);
     
      v3.setInputSplitSource(source1);
      v5.setInputSplitSource(source2);
     
      List<AbstractJobVertex> ordered = new ArrayList<AbstractJobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

      ExecutionGraph eg = new ExecutionGraph(jobId, jobName, cfg);
      try {
        eg.attachJobGraph(ordered);
      }
      catch (JobException e) {
        e.printStackTrace();
        fail("Job failed with exception: " + e.getMessage());
      }
     
      assertEquals(assigner1, eg.getAllVertices().get(v3.getID()).getSplitAssigner());
      assertEquals(assigner2, eg.getAllVertices().get(v5.getID()).getSplitAssigner());
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
View Full Code Here


    try {
      final JobID jobId = new JobID();
      final String jobName = "Test Job Sample Name";
      final Configuration cfg = new Configuration();
     
      AbstractJobVertex v1 = new AbstractJobVertex("vertex1");
      AbstractJobVertex v2 = new AbstractJobVertex("vertex2");
      AbstractJobVertex v3 = new AbstractJobVertex("vertex3");
     
      v1.setParallelism(5);
      v2.setParallelism(7);
      v3.setParallelism(2);

      IntermediateDataSet result = v1.createAndAddResultDataSet();
      v2.connectDataSetAsInput(result, DistributionPattern.BIPARTITE);
      v3.connectDataSetAsInput(result, DistributionPattern.BIPARTITE);
     
      List<AbstractJobVertex> ordered = new ArrayList<AbstractJobVertex>(Arrays.asList(v1, v2, v3));

      ExecutionGraph eg = new ExecutionGraph(jobId, jobName, cfg);
View Full Code Here

      final JobID jobId = new JobID();
      final String jobName = "Co-Location Constraint Sample Job";
      final Configuration cfg = new Configuration();
     
      // simple group of two, cyclic
      AbstractJobVertex v1 = new AbstractJobVertex("vertex1");
      AbstractJobVertex v2 = new AbstractJobVertex("vertex2");
      v1.setParallelism(6);
      v2.setParallelism(4);
     
      SlotSharingGroup sl1 = new SlotSharingGroup();
      v1.setSlotSharingGroup(sl1);
      v2.setSlotSharingGroup(sl1);
      v2.setStrictlyCoLocatedWith(v1);
      v1.setStrictlyCoLocatedWith(v2);
     
      // complex forked dependency pattern
      AbstractJobVertex v3 = new AbstractJobVertex("vertex3");
      AbstractJobVertex v4 = new AbstractJobVertex("vertex4");
      AbstractJobVertex v5 = new AbstractJobVertex("vertex5");
      AbstractJobVertex v6 = new AbstractJobVertex("vertex6");
      AbstractJobVertex v7 = new AbstractJobVertex("vertex7");
      v3.setParallelism(3);
      v4.setParallelism(3);
      v5.setParallelism(3);
      v6.setParallelism(3);
      v7.setParallelism(3);
     
      SlotSharingGroup sl2 = new SlotSharingGroup();
      v3.setSlotSharingGroup(sl2);
      v4.setSlotSharingGroup(sl2);
      v5.setSlotSharingGroup(sl2);
      v6.setSlotSharingGroup(sl2);
      v7.setSlotSharingGroup(sl2);
     
      v4.setStrictlyCoLocatedWith(v3);
      v5.setStrictlyCoLocatedWith(v4);
      v6.setStrictlyCoLocatedWith(v3);
      v3.setStrictlyCoLocatedWith(v7);
     
      // isolated vertex
      AbstractJobVertex v8 = new AbstractJobVertex("vertex8");
      v8.setParallelism(2);
     
      JobGraph jg = new JobGraph(jobId, jobName, v1, v2, v3, v4, v5, v6, v7, v8);
     
      ExecutionGraph eg = new ExecutionGraph(jobId, jobName, cfg);
      eg.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
     
      // check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
      {
        ExecutionVertex[] v1s = eg.getJobVertex(v1.getID()).getTaskVertices();
        ExecutionVertex[] v2s = eg.getJobVertex(v2.getID()).getTaskVertices();
       
        Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
       
        for (int i = 0; i < v2.getParallelism(); i++) {
          assertNotNull(v1s[i].getLocationConstraint());
          assertNotNull(v2s[i].getLocationConstraint());
          assertTrue(v1s[i].getLocationConstraint() == v2s[i].getLocationConstraint());
          all.add(v1s[i].getLocationConstraint());
        }
       
        for (int i = v2.getParallelism(); i < v1.getParallelism(); i++) {
          assertNotNull(v1s[i].getLocationConstraint());
          all.add(v1s[i].getLocationConstraint());
        }
       
        assertEquals("not all co location constraints are distinct", v1.getParallelism(), all.size());
      }
     
      // check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
      {
        ExecutionVertex[] v3s = eg.getJobVertex(v3.getID()).getTaskVertices();
        ExecutionVertex[] v4s = eg.getJobVertex(v4.getID()).getTaskVertices();
        ExecutionVertex[] v5s = eg.getJobVertex(v5.getID()).getTaskVertices();
        ExecutionVertex[] v6s = eg.getJobVertex(v6.getID()).getTaskVertices();
        ExecutionVertex[] v7s = eg.getJobVertex(v7.getID()).getTaskVertices();
       
        Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
       
        for (int i = 0; i < v3.getParallelism(); i++) {
          assertNotNull(v3s[i].getLocationConstraint());
          assertTrue(v3s[i].getLocationConstraint() == v4s[i].getLocationConstraint());
          assertTrue(v4s[i].getLocationConstraint() == v5s[i].getLocationConstraint());
          assertTrue(v5s[i].getLocationConstraint() == v6s[i].getLocationConstraint());
          assertTrue(v6s[i].getLocationConstraint() == v7s[i].getLocationConstraint());
          all.add(v3s[i].getLocationConstraint());
        }
       
        assertEquals("not all co location constraints are distinct", v3.getParallelism(), all.size());
      }
     
      // check the v8 has no co location hints
      {
        ExecutionVertex[] v8s = eg.getJobVertex(v8.getID()).getTaskVertices();
       
        for (int i = 0; i < v8.getParallelism(); i++) {
          assertNull(v8s[i].getLocationConstraint());
        }
      }
    }
    catch (Exception e) {
View Full Code Here

 
  @Test
  public void testScheduleNotEnoughSlots() {
   
    try {
      final AbstractJobVertex vertex = new AbstractJobVertex("Test Vertex");
      vertex.setParallelism(2);
      vertex.setInvokableClass(BlockingNoOpInvokable.class);
     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
     
      final JobManager jm = startJobManager(1);
     
View Full Code Here

  public void testSingleVertexJobImmediately() {
   
    final int NUM_TASKS = 133;
   
    try {
      final AbstractJobVertex vertex = new AbstractJobVertex("Test Vertex");
      vertex.setParallelism(NUM_TASKS);
      vertex.setInvokableClass(NoOpInvokable.class);
     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
View Full Code Here

  public void testSingleVertexJobQueued() {
   
    final int NUM_TASKS = 111;
   
    try {
      final AbstractJobVertex vertex = new AbstractJobVertex("Test Vertex");
      vertex.setParallelism(NUM_TASKS);
      vertex.setInvokableClass(NoOpInvokable.class);
     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
      jobGraph.setAllowQueuedScheduling(true);
     
      final JobManager jm = startJobManager(10);
View Full Code Here

  public void testForwardJob() {
   
    final int NUM_TASKS = 31;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(Sender.class);
      receiver.setInvokableClass(Receiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
 
View Full Code Here

  public void testBipartiteJob() {
   
    final int NUM_TASKS = 31;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(Sender.class);
      receiver.setInvokableClass(AgnosticReceiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
 
View Full Code Here

  public void testTwoInputJobFailingEdgeMismatch() {
   
    final int NUM_TASKS = 11;
   
    try {
      final AbstractJobVertex sender1 = new AbstractJobVertex("Sender1");
      final AbstractJobVertex sender2 = new AbstractJobVertex("Sender2");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender1.setInvokableClass(Sender.class);
      sender2.setInvokableClass(Sender.class);
      receiver.setInvokableClass(AgnosticReceiver.class);
     
      sender1.setParallelism(NUM_TASKS);
      sender2.setParallelism(2*NUM_TASKS);
      receiver.setParallelism(3*NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE);
      receiver.connectNewDataSetAsInput(sender2, DistributionPattern.BIPARTITE);
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      final JobManager jm = startJobManager(6*NUM_TASKS);
     
 
View Full Code Here

  public void testAccumulatedStateFinished() {
    try {
      final JobID jid = new JobID();
      final JobVertexID vid = new JobVertexID();
     
      AbstractJobVertex ajv = new AbstractJobVertex("TestVertex", vid);
      ajv.setParallelism(3);
      ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());
     
      ExecutionGraph graph = new ExecutionGraph(jid, "test job", new Configuration());
      graph.attachJobGraph(Arrays.asList(ajv));
     
      setGraphStatus(graph, JobStatus.RUNNING);
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.jobgraph.AbstractJobVertex

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.