Package org.apache.tez.runtime.api

Examples of org.apache.tez.runtime.api.InputInitializerContext

69
70
71
72
73
74
75
76
77
78
79
    payloadProto.setSplits(splitsProtoBuilder.build());
    payloadProto.setConfigurationBytes(confByteString);
    UserPayload userPayload =
        UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());

    InputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
    MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);

    List<Event> events = splitDist.initialize();

    assertEquals(3, events.size());
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
    payloadProto.setSplits(splitsProtoBuilder.build());
    payloadProto.setConfigurationBytes(confByteString);
    UserPayload userPayload =
        UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());

    InputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
    MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);

    List<Event> events = splitDist.initialize();

    assertEquals(3, events.size());
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
 
  public void runInputInitializers(List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>
      inputs) {
    for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) {

      InputInitializerContext context =
          new TezRootInputInitializerContextImpl(input, vertex, appContext);
      InputInitializer initializer = createInitializer(input, context);

      InitializerWrapper initializerWrapper = new InitializerWrapper(input, initializer, context, vertex);
      initializerMap.put(input.getName(), initializerWrapper);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
    this(null);
  }

  @Override
  public List<Event> initialize() throws Exception {
    InputInitializerContext rootInputContext = getContext();

    context = rootInputContext;

    MRInputUserPayloadProto userPayloadProto =
        MRInputHelpers.parseMRInputPayload(rootInputContext.getInputUserPayload());

    Configuration conf =
        TezUtils.createConfFromByteString(userPayloadProto.getConfigurationBytes());

    boolean sendSerializedEvents =
        conf.getBoolean("mapreduce.tez.input.initializer.serialize.event.payload", true);

    // Read all credentials into the credentials instance stored in JobConf.
    JobConf jobConf = new JobConf(conf);
    ShimLoader.getHadoopShims().getMergedCredentials(jobConf);

    MapWork work = Utilities.getMapWork(jobConf);

    // perform dynamic partition pruning
    pruner.prune(work, jobConf, context);

    InputSplitInfoMem inputSplitInfo = null;
    String realInputFormatName = conf.get("mapred.input.format.class");
    boolean groupingEnabled = userPayloadProto.getGroupingEnabled();
    if (groupingEnabled) {
      // Need to instantiate the realInputFormat
      InputFormat<?, ?> inputFormat =
          (InputFormat<?, ?>) ReflectionUtils.newInstance(Class.forName(realInputFormatName),
              jobConf);

      int totalResource = rootInputContext.getTotalAvailableResource().getMemory();
      int taskResource = rootInputContext.getVertexTaskResource().getMemory();
      int availableSlots = totalResource / taskResource;

      // Create the un-grouped splits
      float waves =
          conf.getFloat(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES,
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.api.InputInitializerContext

  • org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator
  • org.apache.tez.dag.app.dag.RootInputInitializerManager
  • org.apache.tez.mapreduce.common.TestMRInputSplitDistributor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.