Package org.apache.hcatalog.data.schema

Examples of org.apache.hcatalog.data.schema.HCatSchema

92
93
94
95
96
97
98
    fields.add(new FieldSchema("adouble", "double", ""));
    fields.add(new FieldSchema("astring", "string", ""));
    fields.add(new FieldSchema("anullint", "int", ""));
    fields.add(new FieldSchema("anullstring", "string", ""));

    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields));
  }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
    }

    String inputJobInfoStr = conf.get(HCatConstants.HCAT_KEY_JOB_INFO);
    jobInfo =
      (InputJobInfo) HCatUtil.deserialize(inputJobInfoStr);
    HCatSchema tableSchema = jobInfo.getTableInfo().getDataColumns();
    HCatSchema partitionSchema =
      jobInfo.getTableInfo().getPartitionColumns();
    hCatFullTableSchema = new HCatSchema(tableSchema.getFields());
    for (HCatFieldSchema hfs : partitionSchema.getFields()) {
      hCatFullTableSchema.append(hfs);
    }
    hCatSchemaFields = hCatFullTableSchema.getFields();

  }
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
      for (int i = dataFieldsCount; i < totalFieldsCount; ++i) {
        hCatPartitionSchemaFields.add(hCatFullTableSchema.get(i));
      }
    }

    hCatPartitionSchema = new HCatSchema(hCatPartitionSchemaFields);
    for (HCatFieldSchema hfs : hCatPartitionSchemaFields) {
      if (hfs.getType() != HCatFieldSchema.Type.STRING) {
        throw new IOException("The table provided "
          + getQualifiedHCatTableName()
          + " uses unsupported  partitioning key type  for column "
          + hfs.getName() + " : " + hfs.getTypeString() + ".  Only string "
          + "fields are allowed in partition columns in HCatalog");
      }

    }
    LOG.info("HCatalog table partitioning key fields = "
      + Arrays.toString(hCatPartitionSchema.getFieldNames().toArray()));

    List<HCatFieldSchema> outputFieldList = new ArrayList<HCatFieldSchema>();
    for (String col : dbColumnNames) {
      HCatFieldSchema hfs = hCatFullTableSchema.get(col);
      if (hfs == null) {
        throw new IOException("Database column " + col + " not found in "
          + " hcatalog table.");
      }
      if (hCatStaticPartitionKey != null
        && hCatStaticPartitionKey.equals(col)) {
        continue;
      }
      outputFieldList.add(hCatFullTableSchema.get(col));
    }

    projectedSchema = new HCatSchema(outputFieldList);

    LOG.info("HCatalog projected schema fields = "
      + Arrays.toString(projectedSchema.getFieldNames().toArray()));

    validateStaticPartitionKey();
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
      }
    }
    storerInfoStr.append("\n");
    LOG.info(storerInfoStr);

    hCatFullTableSchema = new HCatSchema(dataColsSchema.getFields());
    for (HCatFieldSchema hfs : partitionSchema.getFields()) {
      hCatFullTableSchema.append(hfs);
    }
    fieldCount = hCatFullTableSchema.size();
    lobLoader = new LargeObjectLoader(conf,
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
        throw new CatalogException(e);
      }

      // convert hcatalog field schema into tajo field schema.
      schema = new org.apache.tajo.catalog.Schema();
      HCatSchema tableSchema = null;

      try {
        tableSchema = HCatUtil.getTableSchemaWithPtnCols(table);
      } catch (IOException ioe) {
        throw new CatalogException("Fail to get table schema. - tableName:" + tableName, ioe);
      }
      List<HCatFieldSchema> fieldSchemaList = tableSchema.getFields();
      boolean isPartitionKey = false;
      for (HCatFieldSchema eachField : fieldSchemaList) {
        isPartitionKey = false;

        if (table.getPartitionKeys() != null) {
          for (FieldSchema partitionKey : table.getPartitionKeys()) {
            if (partitionKey.getName().equals(eachField.getName())) {
              isPartitionKey = true;
            }
          }
        }

        if (!isPartitionKey) {
          String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName +
              CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName();
          TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(eachField.getType().toString());
          schema.addColumn(fieldName, dataType);
        }
      }

      // validate field schema.
      try {
        HCatalogUtil.validateHCatTableAndTajoSchema(tableSchema);
      } catch (Exception e) {
        throw new CatalogException("HCatalog cannot support schema. - schema:" + tableSchema.toString(), e);
      }

      stats = new TableStats();
      options = new Options();
      options.putAll(table.getParameters());
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    @Test
    public void testGetWithName() throws Exception {
        TypeInfo ti = getTypeInfo();
        HCatRecord r = new LazyHCatRecord(getHCatRecord(), getObjectInspector(ti));
        HCatSchema schema = HCatSchemaUtils.getHCatSchema(ti)
                                              .get(0).getStructSubSchema();
        Assert.assertEquals(INT_CONST, ((Integer) r.get("an_int", schema)).intValue());
        Assert.assertEquals(LONG_CONST, ((Long) r.get("a_long", schema)).longValue());
        Assert.assertEquals(DOUBLE_CONST, ((Double) r.get("a_double", schema)).doubleValue(), 0);
        Assert.assertEquals(STRING_CONST, (String) r.get("a_string", schema));
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
        Table table = new Table(apiTable);

        List<HCatFieldSchema> expectedHCatSchema =
                Lists.newArrayList(new HCatFieldSchema("username", HCatFieldSchema.Type.STRING, null));

        Assert.assertEquals(new HCatSchema(expectedHCatSchema),
                HCatUtil.getTableSchemaWithPtnCols(table));

        // Add a partition key & ensure its reflected in the schema.
        List<FieldSchema> partitionKeys =
                Lists.newArrayList(new FieldSchema("dt", serdeConstants.STRING_TYPE_NAME, null));
        table.getTTable().setPartitionKeys(partitionKeys);
        expectedHCatSchema.add(new HCatFieldSchema("dt", HCatFieldSchema.Type.STRING, null));
        Assert.assertEquals(new HCatSchema(expectedHCatSchema),
                HCatUtil.getTableSchemaWithPtnCols(table));
    }
View Full Code Here
175
176
177
178
179
180
181
182
        List<HCatFieldSchema> expectedHCatSchema = Lists.newArrayList(
                new HCatFieldSchema("myint", HCatFieldSchema.Type.INT, null),
                new HCatFieldSchema("mystring", HCatFieldSchema.Type.STRING, null),
                new HCatFieldSchema("underscore_int", HCatFieldSchema.Type.INT, null));

        Assert.assertEquals(new HCatSchema(expectedHCatSchema),
                HCatUtil.getTableSchemaWithPtnCols(table));
    }
View Full Code Here
406
407
408
409
410
411
412
413
414
415
416
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            for (int i = 1; i < vals.length; i++) {
                String pair[] = vals[i].split(":");
                record.set(pair[0], schema, pair[1]);
View Full Code Here
451
452
453
454
455
456
457
458
459
460
461
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            synchronized (MapWriteAbortTransaction.class) {
                if (count == 2) {
                    failedKey = vals[0];
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.data.schema.HCatSchema

  • org.apache.giraph.io.hcatalog.GiraphHCatInputFormat
  • org.apache.hcatalog.common.HCatUtil
  • org.apache.hcatalog.common.TestHCatUtil
  • org.apache.hcatalog.data.JsonSerDe
  • org.apache.hcatalog.data.schema.TestHCatSchema
  • org.apache.hcatalog.data.schema.TestHCatSchemaUtils
  • org.apache.hcatalog.data.TestDefaultHCatRecord
  • org.apache.hcatalog.data.TestLazyHCatRecord
  • org.apache.hcatalog.hbase.HBaseHCatStorageHandler
  • org.apache.hcatalog.hbase.HBaseRevisionManagerUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.