Package org.apache.hadoop.hive.ql.plan

Examples of org.apache.hadoop.hive.ql.plan.MapredWork

786
787
788
789
790
791
792
793
    return getWork().getMapWork().getAliasToWork().values();
  }

  @Override
  public boolean hasReduce() {
    MapredWork w = getWork();
    return w.getReduceWork() != null;
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
      LinkedHashMap<String, Operator<? extends OperatorDesc>> ao =
        new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
      ao.put("a", op);

      MapredWork mrwork = new MapredWork();
      mrwork.getMapWork().setPathToAliases(pa);
      mrwork.getMapWork().setPathToPartitionInfo(pt);
      mrwork.getMapWork().setAliasToWork(ao);

      // serialize the configuration once ..
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      Utilities.serializeObject(mrwork, baos);
      baos.close();
      String v1 = baos.toString();

      // store into configuration
      JobConf job = new JobConf(TestPlan.class);
      job.set("fs.default.name", "file:///");
      Utilities.setMapRedWork(job, mrwork, System.getProperty("java.io.tmpdir") + File.separator +
        System.getProperty("user.name") + File.separator + "hive");
      MapredWork mrwork2 = Utilities.getMapRedWork(job);
      Utilities.clearWork(job);

      // over here we should have some checks of the deserialized object against
      // the orginal object
      // System.out.println(v1);
View Full Code Here
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
        new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
      aliasToWork.put("a", cdop1);
      aliasToWork.put("b", cdop2);

      // initialize mapredWork
      MapredWork mrwork = new MapredWork();
      mrwork.getMapWork().setPathToAliases(pathToAliases);
      mrwork.getMapWork().setPathToPartitionInfo(pathToPartitionInfo);
      mrwork.getMapWork().setAliasToWork(aliasToWork);

      // get map operator and initialize it
      MapOperator mo = new MapOperator();
      mo.initializeAsRoot(hconf, mrwork.getMapWork());

      Text tw = new Text();
      InspectableObject io1 = new InspectableObject();
      InspectableObject io2 = new InspectableObject();
      for (int i = 0; i < 5; i++) {
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
      String hiveConfArgs = generateCmdLine(conf);

      // write out the plan to a local file
      Path planPath = new Path(ctx.getLocalTmpFileURI(), "plan.xml");
      OutputStream out = FileSystem.getLocal(conf).create(planPath);
      MapredWork plan = getWork();
      LOG.info("Generating plan file " + planPath.toString());
      Utilities.serializeMapRedWork(plan, out);

      String isSilent = "true".equalsIgnoreCase(System
          .getProperty("test.silent")) ? "-nolog" : "";
View Full Code Here
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
  // loop over all the tasks recursviely
  private void setKeyDescTaskTree(Task<? extends Serializable> task) {

    if (task instanceof ExecDriver) {
      MapredWork work = (MapredWork) task.getWork();
      work.deriveExplainAttributes();
      HashMap<String, Operator<? extends Serializable>> opMap = work
          .getAliasToWork();
      if (!opMap.isEmpty()) {
        for (Operator<? extends Serializable> op : opMap.values()) {
          GenMapRedUtils.setKeyAndValueDesc(work, op);
        }
View Full Code Here
656
657
658
659
660
661
662
663
664
665
666
      MapredLocalWork plan = Utilities.deserializeMapRedLocalWork(pathData, conf);
      MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent);
      ret = ed.executeFromChildJVM(new DriverContext());

    } else {
      MapredWork plan = Utilities.deserializeMapRedWork(pathData, conf);
      ExecDriver ed = new ExecDriver(plan, conf, isSilent);
      ret = ed.execute(new DriverContext());
    }

    if (ret != 0) {
View Full Code Here
726
727
728
729
730
731
732
733
    return true;
  }

  @Override
  public boolean hasReduce() {
    MapredWork w = getWork();
    return w.getReducer() != null;
  }
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
175
176
      // modify the parse context to use indexing
      // we need to delay this until we choose one index so that we don't attempt to modify pctx multiple times
      HiveIndexQueryContext queryContext = queryContexts.get(chosenIndex);

      // prepare the map reduce job to use indexing
      MapredWork work = currentTask.getWork();
      work.setInputformat(queryContext.getIndexInputFormat());
      work.setIndexIntermediateFile(queryContext.getIndexIntermediateFile());

      // modify inputs based on index query
      Set<ReadEntity> inputs = pctx.getSemanticInputs();
      inputs.addAll(queryContext.getAdditionalSemanticInputs());
      List<Task<?>> chosenRewrite = queryContext.getQueryTasks();
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
        if ((mvTask != null) && !mvTask.isLocal()) {
          // There are separate configuration parameters to control whether to
          // merge for a map-only job
          // or for a map-reduce job
          MapredWork currWork = (MapredWork) currTask.getWork();
          boolean mergeMapOnly =
            hconf.getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPFILES) &&
            currWork.getReducer() == null;
          boolean mergeMapRed =
            hconf.getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPREDFILES) &&
            currWork.getReducer() != null;
          if (mergeMapOnly || mergeMapRed) {
            chDir = true;
          }
        }
      }
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
  private void addStatsTask(FileSinkOperator nd, MoveTask mvTask,
      Task<? extends Serializable> currTask, HiveConf hconf) {

    MoveWork mvWork = ((MoveTask)mvTask).getWork();
    StatsWork statsWork = new StatsWork(mvWork.getLoadTableWork());
    MapredWork mrWork = (MapredWork) currTask.getWork();

    // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix
    // in FileSinkDesc is used for stats publishing. They should be consistent.
    statsWork.setAggKey(((FileSinkOperator)nd).getConf().getStatsAggPrefix());
    Task<? extends Serializable> statsTask = TaskFactory.get(statsWork, hconf);

    // mark the MapredWork and FileSinkOperator for gathering stats
    nd.getConf().setGatherStats(true);
    mrWork.setGatheringStats(true);
    // mrWork.addDestinationTable(nd.getConf().getTableInfo().getTableName());

    // subscribe feeds from the MoveTask so that MoveTask can forward the list
    // of dynamic partition list to the StatsTask
    mvTask.addDependentTask(statsTask);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.plan.MapredWork

  • com.sap.hadoop.windowing.runtime2.mr.MRExecutor
  • com.twitter.ambrose.hive.HiveDAGTransformer
  • org.apache.hadoop.hive.ql.exec.ExecDriver
  • org.apache.hadoop.hive.ql.exec.ExecMapper
  • org.apache.hadoop.hive.ql.exec.ExecReducer
  • org.apache.hadoop.hive.ql.exec.MapRedTask
  • org.apache.hadoop.hive.ql.exec.MoveTask
  • org.apache.hadoop.hive.ql.exec.mr.ExecDriver
  • org.apache.hadoop.hive.ql.exec.mr.MapRedTask
  • org.apache.hadoop.hive.ql.exec.TestOperators
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.