Package org.apache.hadoop.hive.ql.plan

Examples of org.apache.hadoop.hive.ql.plan.MapredLocalWork

500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
      assert plan.getAliasToWork().get(alias_id) == null;
      plan.getAliasToWork().put(alias_id, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias_id) == null;
      assert localPlan.getAliasToFetchWork().get(alias_id) == null;
      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null)
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(fetchWork.convertPathToStringArray(partDir), partDesc));
      else
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(tblDir.toString(), tblDesc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
      plan.getPathToPartitionInfo().put(path, new partitionDesc(tt_desc, null));
      plan.getAliasToWork().put(alias, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
                                        new LinkedHashMap<String, Operator<? extends Serializable>>(),
                                        new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias) == null;
      assert localPlan.getAliasToFetchWork().get(alias) == null;
      localPlan.getAliasToWork().put(alias, topOp);
      localPlan.getAliasToFetchWork().put(alias, new fetchWork(alias, tt_desc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      // initialize map local operators
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        // All the operators need to be initialized before process
        forwardOp.initialize(jc, new ObjectInspector[]{entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
      // defer processing of map local operators to first row if in case there is no input (??)
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      mo.setOutputCollector(oc);
      mo.setReporter(rp);
      // process map local operators
      if (fetchOperators != null) {
        try {
          mapredLocalWork localWork = mo.getConf().getMapLocalWork();
          int fetchOpNum = 0;
          for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
            int fetchOpRows = 0;
            String alias = entry.getKey();
            FetchOperator fetchOp = entry.getValue();
            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

            while (true) {
              InspectableObject row = fetchOp.getNextRow();
              if (row == null) {
                forwardOp.close(false);
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
219
220
    // detecting failed executions by exceptions thrown by the operator tree
    // ideally hadoop should let us know whether map execution failed or not
    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }
     
      if (l4j.isInfoEnabled()) {
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      // initialize map local operators
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        // All the operators need to be initialized before process
        forwardOp.initialize(jc, new ObjectInspector[]{entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
      // defer processing of map local operators to first row if in case there is no input (??)
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      mo.setOutputCollector(oc);
      mo.setReporter(rp);
      // process map local operators
      if (fetchOperators != null) {
        try {
          mapredLocalWork localWork = mo.getConf().getMapLocalWork();
          int fetchOpNum = 0;
          for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
            int fetchOpRows = 0;
            String alias = entry.getKey();
            FetchOperator fetchOp = entry.getValue();
            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

            while (true) {
              InspectableObject row = fetchOp.getNextRow();
              if (row == null) {
                break;
View Full Code Here
202
203
204
205
206
207
208
209
210
211
212
213
214
    // detecting failed executions by exceptions thrown by the operator tree
    // ideally hadoop should let us know whether map execution failed or not
    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }
     
      if (l4j.isInfoEnabled()) {
View Full Code Here
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
      assert plan.getAliasToWork().get(alias_id) == null;
      plan.getAliasToWork().put(alias_id, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias_id) == null;
      assert localPlan.getAliasToFetchWork().get(alias_id) == null;
      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null)
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(fetchWork.convertPathToStringArray(partDir), partDesc));
      else
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(tblDir.toString(), tblDesc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
      plan.getPathToPartitionInfo().put(path, new partitionDesc(tt_desc, null));
      plan.getAliasToWork().put(alias, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
                                        new LinkedHashMap<String, Operator<? extends Serializable>>(),
                                        new LinkedHashMap<String, fetchWork>());
     
      assert localPlan.getAliasToWork().get(alias) == null;
      assert localPlan.getAliasToFetchWork().get(alias) == null;
      localPlan.getAliasToWork().put(alias, topOp);
      localPlan.getAliasToFetchWork().put(alias, new fetchWork(alias, tt_desc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.plan.MapredLocalWork

  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hive.ql.exec.ExecDriver
  • org.apache.hadoop.hive.ql.exec.ExecMapper
  • org.apache.hadoop.hive.ql.exec.MapredLocalTask
  • org.apache.hadoop.hive.ql.exec.mr.ExecDriver
  • org.apache.hadoop.hive.ql.exec.mr.ExecMapper
  • org.apache.hadoop.hive.ql.exec.mr.HashTableLoader
  • org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
  • org.apache.hadoop.hive.ql.exec.Utilities
  • org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.