Package org.apache.tez.runtime.api.impl

Examples of org.apache.tez.runtime.api.impl.InputSpec

121
122
123
124
125
126
127
128
129
130
131
    Path mapInput = new Path(workDir, "map0");
   
   
    MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput);

    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        InputDescriptor.create(MRInputLegacy.class.getName())
            .setUserPayload(UserPayload.create(ByteBuffer.wrap(
                MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
                    .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build()
                    .toByteArray()))),
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
   
    Path mapInput = new Path(workDir, "map0");
    MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput);

    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        InputDescriptor.create(MRInputLegacy.class.getName())
            .setUserPayload(UserPayload.create(ByteBuffer.wrap(
                MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
                    .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build()
                    .toByteArray()))),
        1);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex",
        OutputDescriptor.create(LocalOnFileSorterOutput.class.getName()).
          setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
    // Run a map
    LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0,
        mapInput, new TestUmbilical(), dagName, mapVertexName,
        Collections.singletonList(mapInputSpec),
        Collections.singletonList(mapOutputSpec));

    mapTask.initialize();
    mapTask.run();
    mapTask.close();
   
    LOG.info("Starting reduce...");
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();

    jobConf.setOutputFormat(SequenceFileOutputFormat.class);
    jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
        "localized-resources").toUri().toString());
    FileOutputFormat.setOutputPath(jobConf, new Path(workDir, "output"));
    ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create(
        ReduceProcessor.class.getName()).setUserPayload(
        TezUtils.createUserPayloadFromConf(jobConf));
   
    InputSpec reduceInputSpec = new InputSpec(mapVertexName,
        InputDescriptor.create(LocalMergedInput.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        OutputDescriptor.create(MROutputLegacy.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
View Full Code Here
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
      locationHint = TaskLocationHint.createTaskLocationHint(hostsSet, racksSet);
    }
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "dagName", "vertexName", -1,
        ProcessorDescriptor.create("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
          InputDescriptor.create("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
          OutputDescriptor.create("outputClassName"), 1)), null), ta, locationHint,
      priority, containerContext);
    return lr;
View Full Code Here
121
122
123
124
125
126
127
128
    return taskSpec;
  }

  private List<InputSpec> createInputSpecList() {
    InputDescriptor inputDesc = InputDescriptor.create(TestInput.class.getName());
    InputSpec inputSpec = new InputSpec("inedge", inputDesc, 1);
    return Lists.newArrayList(inputSpec);
  }
View Full Code Here
103
104
105
106
107
108
109
110
    return taskSpec;
  }

  private List<InputSpec> createInputSpecList() {
    InputDescriptor inputDesc = new InputDescriptor(TestInput.class.getName());
    InputSpec inputSpec = new InputSpec("inedge", inputDesc, 1);
    return Lists.newArrayList(inputSpec);
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
    Path mapInput = new Path(workDir, "map0");
   
   
    MapUtils.generateInputSplit(localFs, workDir, job, mapInput);
   
    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        new InputDescriptor(MRInputLegacy.class.getName())
            .setUserPayload(MRHelpers.createMRInputPayload(job, null)),
        0);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", new OutputDescriptor(LocalOnFileSorterOutput.class.getName()), 1);
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    mapConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
   
    Path mapInput = new Path(workDir, "map0");
    MapUtils.generateInputSplit(localFs, workDir, mapConf, mapInput);
   
    InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
        new InputDescriptor(MRInputLegacy.class.getName())
            .setUserPayload(MRHelpers.createMRInputPayload(mapConf, null)),
        0);
    OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", new OutputDescriptor(LocalOnFileSorterOutput.class.getName()), 1);
    // Run a map
    LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, mapConf, 0,
        mapInput, new TestUmbilical(), mapVertexName,
        Collections.singletonList(mapInputSpec),
        Collections.singletonList(mapOutputSpec));

    mapTask.initialize();
    mapTask.run();
    mapTask.close();
   
    LOG.info("Starting reduce...");
   
    Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>();
   
    Configuration reduceStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
        reduceVertexName);
    JobConf reduceConf = new JobConf(reduceStageConf);
    reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
    reduceConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
        "localized-resources").toUri().toString());
    FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
    ProcessorDescriptor reduceProcessorDesc = new ProcessorDescriptor(
        ReduceProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(reduceConf));
   
    InputSpec reduceInputSpec = new InputSpec(mapVertexName,
        new InputDescriptor(LocalMergedInput.class.getName()), 1);
    OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
        new OutputDescriptor(MROutputLegacy.class.getName()), 1);

    // Now run a reduce
View Full Code Here
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
      this.additionalInputs.put(input.getName(),
          new RootInputLeafOutputDescriptor<InputDescriptor>(input.getName(), id,
              input.hasInitializerClassName() ? input.getInitializerClassName()
                  : null));
      InputSpec inputSpec = new InputSpec(input.getName(), id, 0);
      additionalInputSpecs.add(inputSpec);
    }

  }
View Full Code Here
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
  public synchronized List<InputSpec> getInputSpecList(int taskIndex) {
    inputSpecList = new ArrayList<InputSpec>(
        this.getInputVerticesCount() + additionalInputSpecs.size());
    inputSpecList.addAll(additionalInputSpecs);
    for (Entry<Vertex, Edge> entry : this.getInputVertices().entrySet()) {
      InputSpec inputSpec = entry.getValue().getDestinationSpec(taskIndex);
      if (LOG.isDebugEnabled()) {
        LOG.debug("For vertex : " + this.getName()
            + ", Using InputSpec : " + inputSpec);
      }
      // TODO DAGAM This should be based on the edge type.
View Full Code Here
895
896
897
898
899
900
901
902
903
904
905
    String[] hosts, String[] racks, Priority priority,
    ContainerContext containerContext) {
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
      taID, capability, new TaskSpec(taID, "vertexName",
      new ProcessorDescriptor("processorClassName"),
      Collections.singletonList(new InputSpec("vertexName",
        new InputDescriptor("inputClassName"), 1)),
      Collections.singletonList(new OutputSpec("vertexName",
        new OutputDescriptor("outputClassName"), 1)), null), ta, hosts, racks,
      priority, containerContext);
    return lr;
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.impl.InputSpec

  • org.apache.tez.dag.api.InputDescriptor
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.VertexImpl
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.mapreduce.processor.map.TestMapProcessor
  • org.apache.tez.mapreduce.processor.reduce.TestReduceProcessor
  • org.apache.tez.runtime.TestLogicalIOProcessorRuntimeTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.