Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.InputDescriptor

157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    dist.setJvmMemory(207093760l);

    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1);
   
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(144965632l, e2Callback, e2InputContext2, e2InDesc2);
   
    dist.makeInitialAllocations();

    assertEquals(104857600l, e1Callback.assigned);
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    dist.setJvmMemory(10000l);

    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
   
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
View Full Code Here
220
221
222
223
224
225
226
227
228
      this.assigned = assignedSize;
    }
  }

  protected InputDescriptor createTestInputDescriptor() {
    InputDescriptor desc = mock(InputDescriptor.class);
    doReturn("InputClass").when(desc).getClassName();
    return desc;
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }

  private List<InputSpec> createInputSpecList() {
    InputDescriptor inputDesc = InputDescriptor.create(TestInput.class.getName());
    InputSpec inputSpec = new InputSpec("inedge", inputDesc, 1);
    return Lists.newArrayList(inputSpec);
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
      this.inputValues[i] = -1;
    }
  }

  public static InputDescriptor getInputDesc(UserPayload payload) {
    InputDescriptor desc = InputDescriptor.create(TestInput.class.getName());
    if (payload != null) {
      desc.setUserPayload(payload);
    }
    return desc;
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
                                                                               boolean useLegacyInput) {
    InputSplitInfo inputSplitInfo = null;
    try {
      inputSplitInfo = generateInputSplits(conf, splitsDir);

      InputDescriptor inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class
          .getName() : MRInput.class.getName())
          .setUserPayload(createMRInputPayload(conf, null));
      Map<String, LocalResource> additionalLocalResources = new HashMap<String, LocalResource>();
      updateLocalResourcesForInputSplits(FileSystem.get(conf), inputSplitInfo,
          additionalLocalResources);
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    dist.setJvmMemory(10000l);

    // First request - ScatterGatherShuffleInput
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

    // Second request - BroadcastInput
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

    // Third request - randomOutput (simulates MROutput)
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
    dist.setJvmMemory(10000l);

    // First request - ScatterGatherShuffleInput [weight 6]
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

    // Second request - BroadcastInput [weight 2]
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

    // Third request - randomOutput (simulates MROutput) [weight 1]
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
View Full Code Here
153
154
155
156
157
158
159
160
161
      this.assigned = assignedSize;
    }
  }

  private InputDescriptor createTestInputDescriptor(Class<? extends LogicalInput> inputClazz) {
    InputDescriptor desc = mock(InputDescriptor.class);
    doReturn(inputClazz.getName()).when(desc).getClassName();
    return desc;
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
        createInputSpecList(), createOutputSpecList(), null);
    return taskSpec;
  }

  private List<InputSpec> createInputSpecList() {
    InputDescriptor inputDesc = new InputDescriptor(TestInput.class.getName());
    InputSpec inputSpec = new InputSpec("inedge", inputDesc, 1);
    return Lists.newArrayList(inputSpec);
  }
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.InputDescriptor

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.dag.app.dag.impl.TestDAGImpl
  • org.apache.tez.dag.app.dag.impl.TestRootInputVertexManager
  • org.apache.tez.dag.app.dag.impl.TestVertexScheduler
  • org.apache.tez.dag.app.dag.impl.VertexImpl
  • org.apache.tez.dag.app.dag.impl.VertexImpl$InitTransition
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.