Examples of TableDesc

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
       getNextPath();
       if (currPath == null)
         return null;

       FileInputFormat.setInputPaths(job, currPath);
      tableDesc tmp = currTbl;
      if (tmp == null)
        tmp = currPart.getTableDesc();
       inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
       inputSplits = inputFormat.getSplits(job, 1);    
       splitNum = 0;
      serde = tmp.getDeserializerClass().newInstance();
      serde.initialize(job, tmp.getProperties());
      LOG.debug("Creating fetchTask with deserializer typeinfo: " + serde.getObjectInspector().getTypeName());
      LOG.debug("deserializer properties: " + tmp.getProperties());
      if (!tblDataDone)
        setPrtnDesc();
     }
    
     if (splitNum >= inputSplits.length) {
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    mapredWork gWork = Utilities.getMapRedWork(job);
    reducer = gWork.getReducer();
    reducer.setMapredWork(gWork);
    isTagged = gWork.getNeedsTagging();
    try {
      tableDesc keyTableDesc = gWork.getKeyDesc();
      inputKeyDeserializer = (SerDe)ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), null);
      inputKeyDeserializer.initialize(null, keyTableDesc.getProperties());
      keyObjectInspector = inputKeyDeserializer.getObjectInspector();
      for(int tag=0; tag<gWork.getTagToValueDesc().size(); tag++) {
        // We should initialize the SerDe with the TypeInfo when available.
        tableDesc valueTableDesc = gWork.getTagToValueDesc().get(tag);
        inputValueDeserializer[tag] = (SerDe)ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
        inputValueDeserializer[tag].initialize(null, valueTableDesc.getProperties());
        valueObjectInspector[tag] = inputValueDeserializer[tag].getObjectInspector();
       
        ArrayList<ObjectInspector> ois = new ArrayList<ObjectInspector>();
        ois.add(keyObjectInspector);
        ois.add(valueObjectInspector[tag]);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

470
471
472
473
474
475
476
477
478
479
480
    prop.setProperty("columns", colTypes[0]);
    prop.setProperty("columns.types", colTypes[1]);

    fetchWork fetch = new fetchWork(
      ctx.getResFile(),
      new tableDesc(LazySimpleSerDe.class, TextInputFormat.class, IgnoreKeyTextOutputFormat.class, prop),
      -1
    );   
    fetch.setSerializationNullFormat(" ");
    return TaskFactory.get(fetch, this.conf);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
          continue;
        }

        partitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
        LinkedHashMap<String, String> partSpec = pd.getPartSpec();
        tableDesc td = pd.getTableDesc();
        Properties p = td.getProperties();
        // Add alias, table name, and partitions to hadoop conf
        HiveConf.setVar(hconf, HiveConf.ConfVars.HIVETABLENAME, String.valueOf(p.getProperty("name")));
        HiveConf.setVar(hconf, HiveConf.ConfVars.HIVEPARTITIONNAME, String.valueOf(partSpec));
        try {
          Class sdclass = td.getDeserializerClass();
          if(sdclass == null) {
            String className = td.getSerdeClassName();
            if ((className == "") || (className == null)) {
              throw new HiveException("SerDe class or the SerDe class name is not set for table: " + td.getProperties().getProperty("name"));
            }
            sdclass = MapOperator.class.getClassLoader().loadClass(className);
          }
          deserializer = (Deserializer) sdclass.newInstance();
          deserializer.initialize(hconf, p);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

202
203
204
205
206
207
208
209
210
211
212
213
214
    JobConf newjob = new JobConf(job);
    ArrayList<InputSplit> result = new ArrayList<InputSplit>();

    // for each dir, get the InputFormat, and do getSplits.
    for(Path dir: dirs) {
      tableDesc table = getTableDescFromPath(dir);
      // create a new InputFormat instance if this is the first time to see this class
      Class inputFormatClass = table.getInputFileFormatClass();
      InputFormat inputFormat = getInputFormatFromCache(inputFormatClass);

      FileInputFormat.setInputPaths(newjob, dir);
      newjob.setInputFormat(inputFormat.getClass());
      InputSplit[] iss = inputFormat.getSplits(newjob, numSplits/dirs.length);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

230
231
232
233
234
235
236
237
238
239
240
241
242
    }
    JobConf newjob = new JobConf(job);

    // for each dir, get the InputFormat, and do validateInput.
    for(Path dir: dirs) {
      tableDesc table = getTableDescFromPath(dir);
      // create a new InputFormat instance if this is the first time to see this class
      InputFormat inputFormat = getInputFormatFromCache(table.getInputFileFormatClass());

      FileInputFormat.setInputPaths(newjob, dir);
      newjob.setInputFormat(inputFormat.getClass());
      inputFormat.validateInput(newjob);
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

251
252
253
254
255
256
257
258
259
260
261
    }
    if (partDesc == null) {
      throw new IOException("cannot find dir = " + dir.toString() + " in partToPartitionInfo!");
    }

    tableDesc table = partDesc.getTableDesc();
    if (table == null) {
      throw new IOException("Input " + dir.toString() +
          " does not have associated InputFormat in mapredWork!");
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

153
154
155
156
157
158
159
160
161
162
163
    assert taskTmpDirLst.size() == tt_descLst.size();
    int size = taskTmpDirLst.size();

    for (int pos = 0; pos < size; pos++) {
      String taskTmpDir = taskTmpDirLst.get(pos);
      tableDesc tt_desc = tt_descLst.get(pos);
      if (plan.getPathToAliases().get(taskTmpDir) == null) {
        plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>());
        plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir);
        plan.getPathToPartitionInfo().put(taskTmpDir, new partitionDesc(tt_desc, null));
        plan.getAliasToWork().put(taskTmpDir, currUnionOp);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

371
372
373
374
375
376
377
378
379
380
381
    String taskTmpDir = (new Path(scratchDir + File.separator + randomid + '.' + pathid)).toString();
    pathid++;
    opProcCtx.setPathId(pathid);
   
    Operator<? extends Serializable> parent = op.getParentOperators().get(0);
    tableDesc tt_desc =
      PlanUtils.getBinaryTableDesc(PlanUtils.getFieldSchemasFromRowSchema(parent.getSchema(), "temporarycol"));
   
    // Create a file sink operator for this file name
    Operator<? extends Serializable> fs_op =
      putOpInsertMap(OperatorFactory.get
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.tableDesc

91
92
93
94
95
96
97
98
99
100
101
102
103
104
        sem.setFetchTaskInit(true);
        sem.getFetchTask().initialize(conf);
      }
      FetchTask ft = (FetchTask) sem.getFetchTask();

      tableDesc td = ft.getTblDesc();
      String tableName = "result";
      List<FieldSchema> lst = MetaStoreUtils.getFieldsFromDeserializer(
          tableName, td.getDeserializer());
      String schema = MetaStoreUtils.getDDLFromFieldSchema(tableName, lst);
      return schema;
    }
    return null;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.