Examples of org.apache.howl.data.schema.HowlSchema

org.apache.howl.data.schema.HowlSchema
HowlSchema. This class is NOT thread-safe.

  public static Table getTable(HiveMetaStoreClient client, String dbName, String tableName) throws Exception{
    return client.getTable(dbName,tableName);
  }


  public static HowlSchema getTableSchemaWithPtnCols(Table table) throws IOException{
      HowlSchema tableSchema = extractSchemaFromStorageDescriptor(table.getSd());


      if( table.getPartitionKeys().size() != 0 ) {


        // add partition keys to table schema
        // NOTE : this assumes that we do not ever have ptn keys as columns inside the table schema as well!
        for (FieldSchema fs : table.getPartitionKeys()){
            tableSchema.append(HowlSchemaUtils.getHowlFieldSchema(fs));
        }
      }
      return tableSchema;
    }

View Full Code Here


        // Find out if we need to throw away the tuple or not.
        if(type == DataType.BAG && removeTupleFromBag(tableSchema, fSchema)){
          List<HowlFieldSchema> arrFields = new ArrayList<HowlFieldSchema>(1);
          arrFields.add(getHowlFSFromPigFS(fSchema.schema.getField(0).schema.getField(0)));
          howlFSchema = new HowlFieldSchema(fSchema.alias, Type.ARRAY, new HowlSchema(arrFields), null);
      }
      else{
          howlFSchema = getHowlFSFromPigFS(fSchema);
      }
      fieldSchemas.add(howlFSchema);
      } catch (HowlException he){
          throw new FrontendException(he.getMessage(),PigHowlUtil.PIG_EXCEPTION_CODE,he);
      }
    }


    return new HowlSchema(fieldSchemas);
  }

View Full Code Here


    case DataType.BAG:
      Schema bagSchema = fSchema.schema;
      List<HowlFieldSchema> arrFields = new ArrayList<HowlFieldSchema>(1);
      arrFields.add(getHowlFSFromPigFS(bagSchema.getField(0)));
      return new HowlFieldSchema(fSchema.alias, Type.ARRAY, new HowlSchema(arrFields), "");


    case DataType.TUPLE:
      List<String> fieldNames = new ArrayList<String>();
      List<HowlFieldSchema> howlFSs = new ArrayList<HowlFieldSchema>();
      for( FieldSchema fieldSchema : fSchema.schema.getFields()){
        fieldNames.add( fieldSchema.alias);
        howlFSs.add(getHowlFSFromPigFS(fieldSchema));
      }
      return new HowlFieldSchema(fSchema.alias, Type.STRUCT, new HowlSchema(howlFSs), "");


    case DataType.MAP:{
      // Pig's schema contain no type information about map's keys and
      // values. So, if its a new column assume <string,string> if its existing
      // return whatever is contained in the existing column.
      HowlFieldSchema mapField = getTableCol(fSchema.alias, howlTblSchema);
      HowlFieldSchema valFS;
      List<HowlFieldSchema> valFSList = new ArrayList<HowlFieldSchema>(1);


      if(mapField != null){
        Type mapValType = mapField.getMapValueSchema().get(0).getType();


        switch(mapValType){
        case STRING:
        case BIGINT:
        case INT:
        case FLOAT:
        case DOUBLE:
          valFS = new HowlFieldSchema(fSchema.alias, mapValType, null);
          break;
        default:
          throw new FrontendException("Only pig primitive types are supported as map value types.", PigHowlUtil.PIG_EXCEPTION_CODE);
        }
        valFSList.add(valFS);
        return new HowlFieldSchema(fSchema.alias,Type.MAP,Type.STRING, new HowlSchema(valFSList),"");
      }


      // Column not found in target table. Its a new column. Its schema is map<string,string>
      valFS = new HowlFieldSchema(fSchema.alias, Type.STRING, "");
      valFSList.add(valFS);
      return new HowlFieldSchema(fSchema.alias,Type.MAP,Type.STRING, new HowlSchema(valFSList),"");
     }


    default:
      throw new FrontendException("Unsupported type: "+type+"  in Pig's schema", PigHowlUtil.PIG_EXCEPTION_CODE);
    }

View Full Code Here

      return null;
    }


    Properties props = UDFContext.getUDFContext().getUDFProperties(
        classForUDFCLookup, new String[] {signature});
    HowlSchema howlTableSchema = (HowlSchema) props.get(HowlConstants.HOWL_TABLE_SCHEMA);


    ArrayList<HowlFieldSchema> fcols = new ArrayList<HowlFieldSchema>();
    for(RequiredField rf: fields) {
      fcols.add(howlTableSchema.getFields().get(rf.getIndex()));
    }
    return new HowlSchema(fcols);
  }

View Full Code Here


    storageDriver.setPartitionValues(context, partitionInfo.getPartitionValues());


    //Set the output schema. Use the schema given by user if set, otherwise use the
    //table level schema
    HowlSchema outputSchema = null;
    String outputSchemaString = context.getConfiguration().get(HOWL_KEY_OUTPUT_SCHEMA);
    if( outputSchemaString != null ) {
      outputSchema = (HowlSchema) HowlUtil.deserialize(outputSchemaString);
    } else {
      outputSchema = tableSchema;

View Full Code Here

      // else - this means pig's optimizer never invoked the pushProjection
      // method - so we need all fields and hence we should not call the
      // setOutputSchema on OwlInputFormat
      if (HowlUtil.checkJobContextIfRunningFromBackend(job)){
        try {
          HowlSchema howlTableSchema = (HowlSchema) props.get(HowlConstants.HOWL_TABLE_SCHEMA);
          outputSchema = howlTableSchema;
        } catch (Exception e) {
          throw new IOException(e);
        }
      }

View Full Code Here

  @Override
  public ResourceSchema getSchema(String location, Job job) throws IOException {
    Table table = phutil.getTable(location,
        howlServerUri!=null?howlServerUri:PigHowlUtil.getHowlServerUri(job),
            PigHowlUtil.getHowlServerPrincipal(job));
    HowlSchema howlTableSchema = HowlUtil.getTableSchemaWithPtnCols(table);
    try {
      PigHowlUtil.validateHowlTableSchemaFollowsPigRules(howlTableSchema);
    } catch (IOException e){
      throw new PigException(
          "Table schema incompatible for reading through HowlLoader :" + e.getMessage()
          + ";[Table schema was "+ howlTableSchema.toString() +"]"
          ,PigHowlUtil.PIG_EXCEPTION_CODE, e);
    }
    storeInUDFContext(signature, HowlConstants.HOWL_TABLE_SCHEMA, howlTableSchema);
    outputSchema = howlTableSchema;
    return PigHowlUtil.getResourceSchema(howlTableSchema);

View Full Code Here


        //Handle duplicate publish
        handleDuplicatePublish(job, outputInfo, client, table);


        StorageDescriptor tblSD = table.getSd();
        HowlSchema tableSchema = HowlUtil.extractSchemaFromStorageDescriptor(tblSD);
        StorerInfo storerInfo = InitializeInput.extractStorerInfo(tblSD,table.getParameters());


        List<String> partitionCols = new ArrayList<String>();
        for(FieldSchema schema : table.getPartitionKeys()) {
          partitionCols.add(schema.getName());

View Full Code Here

        // We also need to update the output Schema with these deletions.


        // Note that, output storage drivers never sees partition columns in data
        // or schema.


        HowlSchema schemaWithoutParts = new HowlSchema(schema.getFields());
        for(String partKey : partMap.keySet()){
          Integer idx;
          if((idx = schema.getPosition(partKey)) != null){
            posOfPartCols.add(idx);
            schemaWithoutParts.remove(schema.get(partKey));
          }
        }
        HowlUtil.validatePartitionSchema(jobInfo.getTable(), schemaWithoutParts);
        jobInfo.setPosOfPartCols(posOfPartCols);
        jobInfo.setOutputSchema(schemaWithoutParts);

View Full Code Here

0 1

TOP

Related Classes of org.apache.howl.data.schema.HowlSchema

org.apache.howl.common.HowlUtil

org.apache.howl.mapreduce.HowlInputFormat

org.apache.howl.mapreduce.HowlMapReduceTest

org.apache.howl.mapreduce.HowlOutputFormat

org.apache.howl.mapreduce.TestHowlPartitioned

org.apache.howl.pig.HowlLoader

org.apache.howl.pig.HowlStorer

org.apache.howl.pig.PigHowlUtil

org.apache.howl.rcfile.TestRCFileInputStorageDriver

org.apache.howl.rcfile.TestRCFileOutputStorageDriver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.