Examples of SkewedInfo

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

153
154
155
156
157
158
159
160
161
162
163
164
165
166
      // not support a table with no columns.
      sd.getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
      sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
      sd.setInputFormat(SequenceFileInputFormat.class.getName());
      sd.setOutputFormat(HiveSequenceFileOutputFormat.class.getName());
      SkewedInfo skewInfo = new SkewedInfo();
      skewInfo.setSkewedColNames(new ArrayList<String>());
      skewInfo.setSkewedColValues(new ArrayList<List<String>>());
      skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
      sd.setSkewedInfo(skewInfo);
    }

    org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table();
    {
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
    String newColName = ast.getChild(2).getText();

    /* Validate the operation of renaming a column name. */
    Table tab = getTable(tblName);

    SkewedInfo skewInfo = tab.getTTable().getSd().getSkewedInfo();
    if ((null != skewInfo)
        && (null != skewInfo.getSkewedColNames())
        && skewInfo.getSkewedColNames().contains(oldColName)) {
      throw new SemanticException(oldColName
          + ErrorMsg.ALTER_TABLE_NOT_ALLOWED_RENAME_SKEWED_COLUMN.getMsg());
    }

    AlterTableDesc alterTblDesc = new AlterTableDesc(tblName,
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
      if (!holdDDLTime) {
        Partition newTPart = getPartition(tbl, partSpec, true, newPartPath.toString(),
            inheritTableSpecs);
        if (isSkewedStoreAsSubdir) {
          org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
          SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
          /* Construct list bucketing location mappings from sub-directory name. */
          Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
              newPartPath, skewedInfo);
          /* Add list bucketing location mappings. */
          skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
          newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
          alterPartition(tbl.getTableName(), new Partition(tbl, newCreatedTpart));
          newTPart = getPartition(tbl, partSpec, true, newPartPath.toString(), inheritTableSpecs);
          newCreatedTpart = newTPart.getTPartition();
        }
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

154
155
156
157
158
159
160
161
162
163
164
165
166
167
      // not support a table with no columns.
      sd.getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
      sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
      sd.setInputFormat(SequenceFileInputFormat.class.getName());
      sd.setOutputFormat(HiveSequenceFileOutputFormat.class.getName());
      SkewedInfo skewInfo = new SkewedInfo();
      skewInfo.setSkewedColNames(new ArrayList<String>());
      skewInfo.setSkewedColValues(new ArrayList<List<String>>());
      skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
      sd.setSkewedInfo(skewInfo);
    }

    org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table();
    {
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
        skewedValues = alterTbl.getSkewedColValues();
      }

      if ( null == tbl.getSkewedInfo()) {
        /* Convert non-skewed table to skewed table. */
        SkewedInfo skewedInfo = new SkewedInfo();
        skewedInfo.setSkewedColNames(skewedColNames);
        skewedInfo.setSkewedColValues(skewedValues);
        tbl.setSkewedInfo(skewedInfo);
      } else {
        tbl.setSkewedColNames(skewedColNames);
        tbl.setSkewedColValues(skewedValues);
      }
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
    String newColName = ast.getChild(2).getText();

    /* Validate the operation of renaming a column name. */
    Table tab = getTable(tblName);

    SkewedInfo skewInfo = tab.getTTable().getSd().getSkewedInfo();
    if ((null != skewInfo)
        && (null != skewInfo.getSkewedColNames())
        && skewInfo.getSkewedColNames().contains(oldColName)) {
      throw new SemanticException(oldColName
          + ErrorMsg.ALTER_TABLE_NOT_ALLOWED_RENAME_SKEWED_COLUMN.getMsg());
    }

    AlterTableDesc alterTblDesc = new AlterTableDesc(tblName,
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
   * @return
   */
  private static int calculateListBucketingDMLDepth(StorageDescriptor desc) {
    // list bucketing will introduce more files
    int listBucketingDepth = 0;
    SkewedInfo skewedInfo = desc.getSkewedInfo();
    if ((skewedInfo != null) && (skewedInfo.getSkewedColNames() != null)
        && (skewedInfo.getSkewedColNames().size() > 0)
        && (skewedInfo.getSkewedColValues() != null)
        && (skewedInfo.getSkewedColValues().size() > 0)
        && (skewedInfo.getSkewedColValueLocationMaps() != null)
        && (skewedInfo.getSkewedColValueLocationMaps().size() > 0)) {
      listBucketingDepth = skewedInfo.getSkewedColNames().size();
    }
    return listBucketingDepth;
  }
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
        validate = MetaStoreUtils.validateTblColumns(tbl.getPartitionKeys());
        if (validate != null) {
          throw new InvalidObjectException("Invalid partition column " + validate);
        }
      }
      SkewedInfo skew = tbl.getSd().getSkewedInfo();
      if (skew != null) {
        validate = MetaStoreUtils.validateSkewedColNames(skew.getSkewedColNames());
        if (validate != null) {
          throw new InvalidObjectException("Invalid skew column " + validate);
        }
        validate = MetaStoreUtils.validateSkewedColNamesSubsetCol(
            skew.getSkewedColNames(), tbl.getSd().getCols());
        if (validate != null) {
          throw new InvalidObjectException("Invalid skew column " + validate);
        }
      }
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
    StorageDescriptor sd = new StorageDescriptor(noFS ? null : convertToFieldSchemas(mFieldSchemas),
        msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd
        .isCompressed(), msd.getNumBuckets(), converToSerDeInfo(msd
        .getSerDeInfo()), msd.getBucketCols(), convertToOrders(msd
        .getSortCols()), msd.getParameters());
    SkewedInfo skewedInfo = new SkewedInfo(msd.getSkewedColNames(),
        convertToSkewedValues(msd.getSkewedColValues()),
        covertToSkewedMap(msd.getSkewedColValueLocationMaps()));
    sd.setSkewedInfo(skewedInfo);
    sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories());
    return sd;
View Full Code Here

Examples of org.apache.hadoop.hive.metastore.api.SkewedInfo

527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
   * @return
   */
  private static int calculateListBucketingDMLDepth(StorageDescriptor desc) {
    // list bucketing will introduce more files
    int listBucketingDepth = 0;
    SkewedInfo skewedInfo = desc.getSkewedInfo();
    if ((skewedInfo != null) && (skewedInfo.getSkewedColNames() != null)
        && (skewedInfo.getSkewedColNames().size() > 0)
        && (skewedInfo.getSkewedColValues() != null)
        && (skewedInfo.getSkewedColValues().size() > 0)
        && (skewedInfo.getSkewedColValueLocationMaps() != null)
        && (skewedInfo.getSkewedColValueLocationMaps().size() > 0)) {
      listBucketingDepth = skewedInfo.getSkewedColNames().size();
    }
    return listBucketingDepth;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.