Examples of MapredLocalWork

Examples of org.apache.hadoop.hive.ql.plan.MapredLocalWork

855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
      plan.getPathToAliases().get(path).add(alias);
      plan.getPathToPartitionInfo().put(path, new PartitionDesc(tt_desc, null));
      plan.getAliasToWork().put(alias, topOp);
    } else {
      // populate local work if needed
      MapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null) {
        localPlan = new MapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, FetchWork>());
      }

      assert localPlan.getAliasToWork().get(alias) == null;
      assert localPlan.getAliasToFetchWork().get(alias) == null;
      localPlan.getAliasToWork().put(alias, topOp);
      localPlan.getAliasToFetchWork().put(alias, new FetchWork(alias, tt_desc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
      assert plan.getAliasToWork().get(alias_id) == null;
      plan.getAliasToWork().put(alias_id, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias_id) == null;
      assert localPlan.getAliasToFetchWork().get(alias_id) == null;
      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null)
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(fetchWork.convertPathToStringArray(partDir), partDesc));
      else
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(tblDir.toString(), tblDesc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
      plan.getPathToPartitionInfo().put(path, new partitionDesc(tt_desc, null));
      plan.getAliasToWork().put(alias, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
                                        new LinkedHashMap<String, Operator<? extends Serializable>>(),
                                        new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias) == null;
      assert localPlan.getAliasToFetchWork().get(alias) == null;
      localPlan.getAliasToWork().put(alias, topOp);
      localPlan.getAliasToFetchWork().put(alias, new fetchWork(alias, tt_desc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      // initialize map local operators
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        // All the operators need to be initialized before process
        forwardOp.initialize(jc, new ObjectInspector[]{entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
      // defer processing of map local operators to first row if in case there is no input (??)
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      mo.setOutputCollector(oc);
      mo.setReporter(rp);
      // process map local operators
      if (fetchOperators != null) {
        try {
          mapredLocalWork localWork = mo.getConf().getMapLocalWork();
          int fetchOpNum = 0;
          for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
            int fetchOpRows = 0;
            String alias = entry.getKey();
            FetchOperator fetchOp = entry.getValue();
            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

            while (true) {
              InspectableObject row = fetchOp.getNextRow();
              if (row == null) {
                forwardOp.close(false);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

208
209
210
211
212
213
214
215
216
217
218
219
220
    // detecting failed executions by exceptions thrown by the operator tree
    // ideally hadoop should let us know whether map execution failed or not
    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }
     
      if (l4j.isInfoEnabled()) {
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      mo.initialize(jc, null);

      // initialize map local work
      mapredLocalWork localWork = mrwork.getMapLocalWork();
      if (localWork == null) {
        return;
      }
      fetchOperators = new HashMap<String, FetchOperator>();
      // create map local operators
      for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
        fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
        l4j.info("fetchoperator for " + entry.getKey() + " created");
      }
      // initialize map local operators
      for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
        Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
        // All the operators need to be initialized before process
        forwardOp.initialize(jc, new ObjectInspector[]{entry.getValue().getOutputObjectInspector()});
        l4j.info("fetchoperator for " + entry.getKey() + " initialized");
      }
      // defer processing of map local operators to first row if in case there is no input (??)
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
      mo.setOutputCollector(oc);
      mo.setReporter(rp);
      // process map local operators
      if (fetchOperators != null) {
        try {
          mapredLocalWork localWork = mo.getConf().getMapLocalWork();
          int fetchOpNum = 0;
          for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
            int fetchOpRows = 0;
            String alias = entry.getKey();
            FetchOperator fetchOp = entry.getValue();
            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);

            while (true) {
              InspectableObject row = fetchOp.getNextRow();
              if (row == null) {
                break;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

202
203
204
205
206
207
208
209
210
211
212
213
214
    // detecting failed executions by exceptions thrown by the operator tree
    // ideally hadoop should let us know whether map execution failed or not
    try {
      mo.close(abort);
      if (fetchOperators != null) {
        mapredLocalWork localWork = mo.getConf().getMapLocalWork();
        for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
          forwardOp.close(abort);
        }
      }
     
      if (l4j.isInfoEnabled()) {
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.mapredLocalWork

485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
      assert plan.getAliasToWork().get(alias_id) == null;
      plan.getAliasToWork().put(alias_id, topOp);
    }
    else {
      // populate local work if needed
      mapredLocalWork localPlan = plan.getMapLocalWork();
      if (localPlan == null)
        localPlan = new mapredLocalWork(
            new LinkedHashMap<String, Operator<? extends Serializable>>(),
            new LinkedHashMap<String, fetchWork>());

      assert localPlan.getAliasToWork().get(alias_id) == null;
      assert localPlan.getAliasToFetchWork().get(alias_id) == null;
      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null)
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(fetchWork.convertPathToStringArray(partDir), partDesc));
      else
        localPlan.getAliasToFetchWork().put(alias_id, new fetchWork(tblDir.toString(), tblDesc));
      plan.setMapLocalWork(localPlan);
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.