Package org.apache.tez.runtime.library.common

Examples of org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler

91
92
93
94
95
96
97
98
99
100
101
      LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
          + getContext().getSourceVertexName());
      return Collections.emptyList();
    } else {
      long initalMemReq = getInitialMemoryReq();
      memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
      this.getContext().requestInitialMemory(initalMemReq, memoryUpdateCallbackHandler);
    }

    this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
    this.inputRecordCounter = getContext().getCounters().findCounter(
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
      return Collections.emptyList();
    }

    long initialMemoryRequest = Shuffle.getInitialMemoryRequirement(conf,
        getContext().getTotalMemoryAvailableToTask());
    this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    getContext().requestInitialMemory(initialMemoryRequest, memoryUpdateCallbackHandler);

    this.inputKeyCounter = getContext().getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS);
    this.inputValueCounter = getContext().getCounters().findCounter(
        TaskCounter.REDUCE_INPUT_RECORDS);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
    // Initializing this parametr in this conf since it is used in multiple
    // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
    // TezMerger, etc.
    this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
    this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    getContext().requestInitialMemory(
        ExternalSorter.getInitialMemoryRequirement(conf,
            getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

    sendEmptyPartitionDetails = this.conf.getBoolean(
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
  public synchronized List<Event> initialize() throws Exception {
    this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
    this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
    this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS,
        getNumPhysicalOutputs());
    this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    getContext().requestInitialMemory(
        UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf,
            getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
    return Collections.emptyList();
  }
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
264
265
        .getServiceProviderMetaData
            (ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
    doAnswer(new Answer() {
      @Override public Object answer(InvocationOnMock invocation) throws Throwable {
        long requestedSize = (Long) invocation.getArguments()[0];
        MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
            .getArguments()[1];
        callback.memoryAssigned(requestedSize);
        return null;
      }
    }).when(context).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
    return context;
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    this.conf = TezUtils.createConfFromUserPayload(outputContext.getUserPayload());
    // Initializing this parametr in this conf since it is used in multiple
    // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
    // TezMerger, etc.
    this.conf.setStrings(TezJobConfig.LOCAL_DIRS, outputContext.getWorkDirs());
    this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    outputContext.requestInitialMemory(
        ExternalSorter.getInitialMemoryRequirement(conf,
            outputContext.getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

    sendEmptyPartitionDetails = this.conf.getBoolean(
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
      LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
          + inputContext.getSourceVertexName());
      return Collections.emptyList();
    } else {
      long initalMemReq = getInitialMemoryReq();
      memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
      this.inputContext.requestInitialMemory(initalMemReq, memoryUpdateCallbackHandler);
    }

    this.conf.setStrings(TezJobConfig.LOCAL_DIRS, inputContext.getWorkDirs());
    this.inputRecordCounter = inputContext.getCounters().findCounter(
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
      return Collections.emptyList();
    }
   
    long initialMemoryRequest = Shuffle.getInitialMemoryRequirement(conf,
        inputContext.getTotalMemoryAvailableToTask());
    this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    inputContext.requestInitialMemory(initialMemoryRequest, memoryUpdateCallbackHandler);

    this.inputKeyCounter = inputContext.getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS);
    this.inputValueCounter = inputContext.getCounters().findCounter(
        TaskCounter.REDUCE_INPUT_RECORDS);
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler

  • org.apache.tez.runtime.library.input.OrderedGroupedKVInput
  • org.apache.tez.runtime.library.input.ShuffledMergedInput
  • org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput
  • org.apache.tez.runtime.library.input.UnorderedKVInput
  • org.apache.tez.runtime.library.output.OnFileSortedOutput
  • org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput
  • org.apache.tez.runtime.library.output.TestOnFileSortedOutput
  • org.apache.tez.runtime.library.output.UnorderedPartitionedKVOutput
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.