Package org.apache.hadoop.hive.ql.plan

Examples of org.apache.hadoop.hive.ql.plan.PartitionDesc

178
179
180
181
182
183
184
185
186
187
188
      }
    }

    while (iterPath.hasNext()) {
      Path nxt = iterPath.next();
      partitionDesc prt = iterPartDesc.next();
      FileSystem fs = nxt.getFileSystem(job);
      if (fs.exists(nxt)) {
        FileStatus[] fStats = fs.listStatus(nxt);
        for (FileStatus fStat : fStats) {
          if (fStat.getLen() > 0) {
View Full Code Here
970
971
972
973
974
975
976
977
978
979
980
    if (isEmptyPath) {
      pathToPartitionInfo.put(newPath.toUri().toString(), pathToPartitionInfo.get(path));
      pathToPartitionInfo.remove(path);
    }
    else {
      partitionDesc pDesc = work.getAliasToPartnInfo().get(alias).clone();
      pathToPartitionInfo.put(newPath.toUri().toString(), pDesc);
    }
    work.setPathToPartitionInfo(pathToPartitionInfo);

    String onefile = newPath.toString();
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
    cplan.setReducer(extract);
    ArrayList<String> aliases = new ArrayList<String>();
    aliases.add(fsConf.getDirName());
    cplan.getPathToAliases().put(fsConf.getDirName(), aliases);
    cplan.getAliasToWork().put(fsConf.getDirName(), ts_op);   
    cplan.getPathToPartitionInfo().put(fsConf.getDirName(), new partitionDesc(fsConf.getTableInfo(), null));
    cplan.setNumReduceTasks(-1);
   
    moveWork dummyMv = new moveWork(null, null, null, new loadFileDesc(fsOp.getConf().getDirName(), finalName, true, null, null), false);
    Task<? extends Serializable> dummyMergeTask = TaskFactory.get(dummyMv, ctx.getConf());
    List<Serializable> listWorks = new ArrayList<Serializable>();
View Full Code Here
320
321
322
323
324
325
326
327
328
329
330
      String taskTmpDir = mjCtx.getTaskTmpDir();
      tableDesc tt_desc = mjCtx.getTTDesc();
      assert plan.getPathToAliases().get(taskTmpDir) == null;
      plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>());
      plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir);
      plan.getPathToPartitionInfo().put(taskTmpDir, new partitionDesc(tt_desc, null));
      plan.getAliasToWork().put(taskTmpDir, mjCtx.getRootMapJoinOp());
      return dest;
    }
   
    return dest;
View Full Code Here
244
245
246
247
248
249
250
251
252
253
254
      pathToAliases.put("/testDir", aliases);

      // initialize pathToTableInfo
      // Default: treat the table as a single column "col"
      tableDesc td = Utilities.defaultTd;
      partitionDesc pd = new partitionDesc(td, null);
      LinkedHashMap<String,org.apache.hadoop.hive.ql.plan.partitionDesc> pathToPartitionInfo = new
        LinkedHashMap<String,org.apache.hadoop.hive.ql.plan.partitionDesc> ();
      pathToPartitionInfo.put("/testDir", pd);

      // initialize aliasToWork
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
          Utilities.getMapRedWork(job).getPathToPartitionInfo();

        // extract all the inputFormatClass names for each chunk in the CombinedSplit.
        Path[] ipaths = inputSplitShim.getPaths();
        for (int i = 0; i < ipaths.length; i++) {
          partitionDesc part = null;
          try {
            part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[i].getParent());
          } catch (IOException e) {
            // The file path may be present in case of sampling - so ignore that
            part = null;
          }

          if (part == null) {
            try {
              part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[i]);
            } catch (IOException e) {
              LOG.warn("CombineHiveInputSplit unable to find table description for " +
                       ipaths[i].getParent());
              continue;
            }
          }
         
          // create a new InputFormat instance if this is the first time to see this class
          if (i == 0)
            inputFormatClassName = part.getInputFileFormatClass().getName();
          else
            assert inputFormatClassName.equals(part.getInputFileFormatClass().getName());
        }
      }
    }
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
      if (inputFormatClassName == null) {
        Map<String, partitionDesc> pathToPartitionInfo =
          Utilities.getMapRedWork(getJob()).getPathToPartitionInfo();
       
        // extract all the inputFormatClass names for each chunk in the CombinedSplit.
        partitionDesc part = null;
        try {
          part = getPartitionDescFromPath(pathToPartitionInfo, inputSplitShim.getPath(0).getParent());
        } catch (IOException e) {
          // The file path may be present in case of sampling - so ignore that
          part = null;
        }

        if (part == null)
          part = getPartitionDescFromPath(pathToPartitionInfo, inputSplitShim.getPath(0));

        // create a new InputFormat instance if this is the first time to see this class
        inputFormatClassName = part.getInputFileFormatClass().getName();
      }

      out.writeUTF(inputFormatClassName);
    }
View Full Code Here
232
233
234
235
236
237
238
239
240
241
242
243
244
    JobConf newjob = new JobConf(job);
    ArrayList<InputSplit> result = new ArrayList<InputSplit>();

    // for each dir, get the InputFormat, and do getSplits.
    for(Path dir: dirs) {
      partitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, dir);
      // create a new InputFormat instance if this is the first time to see this class
      Class inputFormatClass = part.getInputFileFormatClass();
      InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);

      FileInputFormat.setInputPaths(newjob, dir);
      newjob.setInputFormat(inputFormat.getClass());
      InputSplit[] iss = inputFormat.getSplits(newjob, numSplits/dirs.length);
View Full Code Here
259
260
261
262
263
264
265
266
267
268
269
270
271
    }
    JobConf newjob = new JobConf(job);

    // for each dir, get the InputFormat, and do validateInput.
    for (Path dir: dirs) {
      partitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, dir);
      // create a new InputFormat instance if this is the first time to see this class
      InputFormat inputFormat = getInputFormatFromCache(part.getInputFileFormatClass(), job);

      FileInputFormat.setInputPaths(newjob, dir);
      newjob.setInputFormat(inputFormat.getClass());
      ShimLoader.getHadoopShims().inputFormatValidateInput(inputFormat, newjob);
    }
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
    }
  }

  protected static partitionDesc getPartitionDescFromPath(Map<String, partitionDesc> pathToPartitionInfo,
                                                  Path dir) throws IOException {
    partitionDesc partDesc = pathToPartitionInfo.get(dir.toString());
    if (partDesc == null) {
      partDesc = pathToPartitionInfo.get(dir.toUri().getPath());
    }
    if (partDesc == null) {
      throw new IOException("cannot find dir = " + dir.toString() + " in partToPartitionInfo!");
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.ql.plan.PartitionDesc

  • com.linkedin.haivvreo.AvroSerDe
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.hive.ql.exec.ExecDriver
  • org.apache.hadoop.hive.ql.exec.FetchOperator
  • org.apache.hadoop.hive.ql.exec.FetchTask
  • org.apache.hadoop.hive.ql.exec.MapOperator
  • org.apache.hadoop.hive.ql.exec.mr.ExecDriver
  • org.apache.hadoop.hive.ql.exec.TestExecDriver
  • org.apache.hadoop.hive.ql.exec.TestOperators
  • org.apache.hadoop.hive.ql.exec.TestPlan
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.