Package org.apache.hadoop.hive.ql.plan.ConditionalResolverSkewJoin

Examples of org.apache.hadoop.hive.ql.plan.ConditionalResolverSkewJoin.ConditionalResolverSkewJoinCtx

158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
            listWork.set(index, (Serializable) localwork);
            conditionalWork.setListWorks(listWork);
            ConditionalResolver resolver = conditionalTask.getResolver();
            if (resolver instanceof ConditionalResolverSkewJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap = context
                  .getDirToTaskMap();
              // to avoid concurrent modify the hashmap
              HashMap<String, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<String, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<String, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
                  .entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                String key = entry.getKey();
                if (task.equals(currTask)) {
                  newbigKeysDirToTaskMap.put(key, localTask);
                } else {
                  newbigKeysDirToTaskMap.put(key, task);
                }
              }
              context.setDirToTaskMap(newbigKeysDirToTaskMap);
              conditionalTask.setResolverCtx(context);
            } else if (resolver instanceof ConditionalResolverCommonJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverCommonJoinCtx context = (ConditionalResolverCommonJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<String, Task<? extends Serializable>> aliasToWork = context.getAliasToTask();
              // to avoid concurrent modify the hashmap
              HashMap<String, Task<? extends Serializable>> newAliasToWork = new HashMap<String, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<String, Task<? extends Serializable>> entry : aliasToWork.entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                String key = entry.getKey();

                if (task.equals(currTask)) {
                  newAliasToWork.put(key, localTask);
                } else {
                  newAliasToWork.put(key, task);
                }
              }
              context.setAliasToTask(newAliasToWork);
              conditionalTask.setResolverCtx(context);
            }
          }
        }
        // make current task depends on this new generated localMapJoinTask
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
            listWork.set(index, localwork);
            conditionalWork.setListWorks(listWork);
            ConditionalResolver resolver = conditionalTask.getResolver();
            if (resolver instanceof ConditionalResolverSkewJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<Path, Task<? extends Serializable>> bigKeysDirToTaskMap = context
                  .getDirToTaskMap();
              // to avoid concurrent modify the hashmap
              HashMap<Path, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<Path, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<Path, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
                  .entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                Path key = entry.getKey();
                if (task.equals(currTask)) {
                  newbigKeysDirToTaskMap.put(key, localTask);
                } else {
                  newbigKeysDirToTaskMap.put(key, task);
                }
              }
              context.setDirToTaskMap(newbigKeysDirToTaskMap);
              conditionalTask.setResolverCtx(context);
            } else if (resolver instanceof ConditionalResolverCommonJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverCommonJoinCtx context = (ConditionalResolverCommonJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<Task<? extends Serializable>, Set<String>> taskToAliases = context.getTaskToAliases();
              // to avoid concurrent modify the hashmap
              HashMap<Task<? extends Serializable>, Set<String>> newTaskToAliases =
                  new HashMap<Task<? extends Serializable>, Set<String>>();
              // reset the resolver
              for (Map.Entry<Task<? extends Serializable>, Set<String>> entry : taskToAliases.entrySet()) {
                Task<? extends Serializable> task = entry.getKey();
                Set<String> key = new HashSet<String>(entry.getValue());

                if (task.equals(currTask)) {
                  newTaskToAliases.put(localTask, key);
                } else {
                  newTaskToAliases.put(task, key);
                }
              }
              context.setTaskToAliases(newTaskToAliases);
              conditionalTask.setResolverCtx(context);
            }
          }
        }
        // make current task depends on this new generated localMapJoinTask
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
361
    }
    if (child != null) {
      currTask.removeDependentTask(child);
      listTasks.add(child);
    }
    ConditionalResolverSkewJoinCtx context =
        new ConditionalResolverSkewJoinCtx(bigKeysDirToTaskMap, child);

    ConditionalWork cndWork = new ConditionalWork(listWorks);
    ConditionalTask cndTsk = (ConditionalTask) TaskFactory.get(cndWork, parseCtx.getConf());
    cndTsk.setListTasks(listTasks);
    cndTsk.setResolver(new ConditionalResolverSkewJoin());
View Full Code Here
348
349
350
351
352
353
354
355
356
357
358
359
    }
    if (child != null) {
      currTask.removeDependentTask(child);
      listTasks.add(child);
    }
    ConditionalResolverSkewJoinCtx context =
        new ConditionalResolverSkewJoinCtx(bigKeysDirToTaskMap, child);

    ConditionalWork cndWork = new ConditionalWork(listWorks);
    ConditionalTask cndTsk = (ConditionalTask) TaskFactory.get(cndWork, parseCtx.getConf());
    cndTsk.setListTasks(listTasks);
    cndTsk.setResolver(new ConditionalResolverSkewJoin());
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
            listWork.set(index, localwork);
            conditionalWork.setListWorks(listWork);
            ConditionalResolver resolver = conditionalTask.getResolver();
            if (resolver instanceof ConditionalResolverSkewJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<Path, Task<? extends Serializable>> bigKeysDirToTaskMap = context
                  .getDirToTaskMap();
              // to avoid concurrent modify the hashmap
              HashMap<Path, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<Path, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<Path, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
                  .entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                Path key = entry.getKey();
                if (task.equals(currTask)) {
                  newbigKeysDirToTaskMap.put(key, localTask);
                } else {
                  newbigKeysDirToTaskMap.put(key, task);
                }
              }
              context.setDirToTaskMap(newbigKeysDirToTaskMap);
              conditionalTask.setResolverCtx(context);
            } else if (resolver instanceof ConditionalResolverCommonJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverCommonJoinCtx context = (ConditionalResolverCommonJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<Task<? extends Serializable>, Set<String>> taskToAliases = context.getTaskToAliases();
              // to avoid concurrent modify the hashmap
              HashMap<Task<? extends Serializable>, Set<String>> newTaskToAliases =
                  new HashMap<Task<? extends Serializable>, Set<String>>();
              // reset the resolver
              for (Map.Entry<Task<? extends Serializable>, Set<String>> entry : taskToAliases.entrySet()) {
                Task<? extends Serializable> task = entry.getKey();
                Set<String> key = new HashSet<String>(entry.getValue());

                if (task.equals(currTask)) {
                  newTaskToAliases.put(localTask, key);
                } else {
                  newTaskToAliases.put(task, key);
                }
              }
              context.setTaskToAliases(newTaskToAliases);
              conditionalTask.setResolverCtx(context);
            }
          }
        }
        // make current task depends on this new generated localMapJoinTask
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
            listWork.set(index, (Serializable) localwork);
            conditionalWork.setListWorks(listWork);
            ConditionalResolver resolver = conditionalTask.getResolver();
            if (resolver instanceof ConditionalResolverSkewJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap = context
                  .getDirToTaskMap();
              // to avoid concurrent modify the hashmap
              HashMap<String, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<String, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<String, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
                  .entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                String key = entry.getKey();
                if (task.equals(currTask)) {
                  newbigKeysDirToTaskMap.put(key, localTask);
                } else {
                  newbigKeysDirToTaskMap.put(key, task);
                }
              }
              context.setDirToTaskMap(newbigKeysDirToTaskMap);
              conditionalTask.setResolverCtx(context);
            } else if (resolver instanceof ConditionalResolverCommonJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverCommonJoinCtx context = (ConditionalResolverCommonJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<String, Task<? extends Serializable>> aliasToWork = context.getAliasToTask();
              // to avoid concurrent modify the hashmap
              HashMap<String, Task<? extends Serializable>> newAliasToWork = new HashMap<String, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<String, Task<? extends Serializable>> entry : aliasToWork.entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                String key = entry.getKey();

                if (task.equals(currTask)) {
                  newAliasToWork.put(key, localTask);
                } else {
                  newAliasToWork.put(key, task);
                }
              }
              context.setAliasToTask(newAliasToWork);
              conditionalTask.setResolverCtx(context);
            }
          }
        }
        // make current task depends on this new generated localMapJoinTask
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
            listWork.set(index, (Serializable) localwork);
            conditionalWork.setListWorks(listWork);
            ConditionalResolver resolver = conditionalTask.getResolver();
            if (resolver instanceof ConditionalResolverSkewJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap = context
                  .getDirToTaskMap();
              // to avoid concurrent modify the hashmap
              HashMap<String, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<String, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<String, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
                  .entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                String key = entry.getKey();
                if (task.equals(currTask)) {
                  newbigKeysDirToTaskMap.put(key, localTask);
                } else {
                  newbigKeysDirToTaskMap.put(key, task);
                }
              }
              context.setDirToTaskMap(newbigKeysDirToTaskMap);
              conditionalTask.setResolverCtx(context);
            } else if (resolver instanceof ConditionalResolverCommonJoin) {
              // get bigKeysDirToTaskMap
              ConditionalResolverCommonJoinCtx context = (ConditionalResolverCommonJoinCtx) conditionalTask
                  .getResolverCtx();
              HashMap<String, Task<? extends Serializable>> aliasToWork = context.getAliasToTask();
              // to avoid concurrent modify the hashmap
              HashMap<String, Task<? extends Serializable>> newAliasToWork = new HashMap<String, Task<? extends Serializable>>();
              // reset the resolver
              for (Map.Entry<String, Task<? extends Serializable>> entry : aliasToWork.entrySet()) {
                Task<? extends Serializable> task = entry.getValue();
                String key = entry.getKey();

                if (task.equals(currTask)) {
                  newAliasToWork.put(key, localTask);
                } else {
                  newAliasToWork.put(key, task);
                }
              }
              context.setAliasToTask(newAliasToWork);
              conditionalTask.setResolverCtx(context);
            }
          }
        }
        // make current task depends on this new generated localMapJoinTask
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.plan.ConditionalResolverSkewJoin.ConditionalResolverSkewJoinCtx

  • org.apache.hadoop.hive.ql.optimizer.physical.GenMRSkewJoinProcessor
  • org.apache.hadoop.hive.ql.optimizer.physical.MapJoinResolver$LocalMapJoinTaskDispatcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.