Examples of com.datasalt.pangool.io.Schema$Field$Props

com.datasalt.pangool.io.Schema
A field is an abstract data type that can be one of this:
- A 32-bit signed int;
- A 64-bit signed long;
- A 32-bit IEEE single-float; or
- A 64-bit IEEE double-float; or
- A unicode string;
- A boolean; or
- An enum, containing one of a small set of symbols;
- A byte buffer
- An arbitrary object, serializable by Hadoop's serialization
A field can be constructed using one of its static createXXX methods. A field object is immutable.

    // serialize common
    write(commonSchema, tuple, commonTranslation, out,serInfo.getCommonSchemaSerializers());
    // serialize source id
    WritableUtils.writeVInt(out, schemaId);
    // serialize rest of the fields
    Schema specificSchema = serInfo.getSpecificSchema(schemaId);
    int[] specificTranslation = serInfo
        .getSpecificSchemaIndexTranslation(schemaId);
    write(specificSchema, tuple, specificTranslation, out,serInfo.getSpecificSchemaSerializers().get(schemaId));
  }

View Full Code Here

    return reuse;
  }


  public void readFields(ITuple tuple, DataInputStream input,FieldDeserializer[] customDeserializers) 
      throws IOException {
    Schema schema = tuple.getSchema();
    for(int index = 0; index < schema.getFields().size(); index++) {
      FieldDeserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);
      switch(field.getType()){
      case INT:  tuple.set(index,WritableUtils.readVInt(input)); break;
      case LONG: tuple.set(index,WritableUtils.readVLong(input)); break;
      case DOUBLE: tuple.set(index,input.readDouble()); break;
      case FLOAT: tuple.set(index,input.readFloat()); break;

View Full Code Here

  }


  private void calculateGroupSchema() {
    List<Field> fields = commonSchema.getFields();
    List<Field> groupFields = fields.subList(0, mrConfig.getGroupByFields().size());
    this.groupSchema = new Schema("group", groupFields);
  }

View Full Code Here

  
  private void initSpecificSchemaSerialization(){
    specificSerializers = new ArrayList<FieldSerializer[]>();
    specificDeserializers = new ArrayList<FieldDeserializer[]>();
    for(int i= 0 ; i < specificSchemas.size(); i++){
      Schema specificSchema = specificSchemas.get(i);
      specificSerializers.add(getSerializers(specificSchema));
      specificDeserializers.add(getDeserializers(specificSchema));
    }
  }

View Full Code Here

      fieldsToPartition.add(posFields);
    }
  }


  private void calculateOneIntermediateCommonSchema() throws TupleMRException {
    Schema intermediateSchema = mrConfig.getIntermediateSchemas().get(0);
    Criteria commonSortCriteria = mrConfig.getCommonCriteria();
    List<Field> commonFields = new ArrayList<Field>();
    for(SortElement sortElement : commonSortCriteria.getElements()) {
      String fieldName = sortElement.getName();
      Field field = checkFieldInAllSchemas(fieldName);
      commonFields.add(Field.cloneField(field, fieldName));
    }


    // adding the rest
    for(Field field : intermediateSchema.getFields()) {
      Map<String,String> aliases = mrConfig.getFieldAliases(intermediateSchema.getName());
      if(!containsField(field.getName(),commonFields,aliases))  {
          commonFields.add(field);
      }
    }
    this.commonSchema = new Schema("common", commonFields);
  }

View Full Code Here

      Field field = checkFieldInAllSchemas(fieldName);
      
      commonFields.add(Field.cloneField(field,fieldName));
    }


    this.commonSchema = new Schema("common", commonFields);
    this.specificSchemas = new ArrayList<Schema>();
    List<List<Field>> specificFieldsBySource = new ArrayList<List<Field>>();


    for(int schemaId = 0; schemaId < mrConfig.getNumIntermediateSchemas(); schemaId++) {
      Criteria specificCriteria = mrConfig.getSpecificOrderBys().get(schemaId);
      List<Field> specificFields = new ArrayList<Field>();
      if(specificCriteria != null) {
        for(SortElement sortElement : specificCriteria.getElements()) {
          String fieldName = sortElement.getName();
          Field field = checkFieldInSchema(fieldName, schemaId);
          specificFields.add(Field.cloneField(field,fieldName));
        }
      }
      specificFieldsBySource.add(specificFields);
    }


    for(int i = 0; i < mrConfig.getNumIntermediateSchemas(); i++) {
      Schema sourceSchema = mrConfig.getIntermediateSchema(i);
      List<Field> specificFields = specificFieldsBySource.get(i);
      for(Field field : sourceSchema.getFields()) {
        Map<String,String> sourceAliases = mrConfig.getFieldAliases(sourceSchema.getName());
        if(!containsField(field.getName(),commonSchema.getFields(),sourceAliases)
            && !containsField(field.getName(), specificFields,sourceAliases)) {
          specificFields.add(field);
        }
      }
      this.specificSchemas.add(new Schema("specific", specificFields));
    }
    this.specificSchemas = Collections.unmodifiableList(this.specificSchemas);
  }

View Full Code Here

    return field;
  }


  private Field checkFieldInSchema(String fieldName, int schemaId)
      throws TupleMRException {
    Schema schema = mrConfig.getIntermediateSchema(schemaId);
    Field field = getFieldUsingAliases(schema,fieldName);
    if(field == null) {
      throw new TupleMRException("Field '" + fieldName + "' not present in source '"
          + schema.getName() + "' " + schema);
    }
    return field;
  }

View Full Code Here

    return groupSchema;
  }


  private void calculateIndexTranslations() {
    for(int schemaId = 0; schemaId < mrConfig.getIntermediateSchemas().size(); schemaId++) {
      Schema sourceSchema = mrConfig.getIntermediateSchema(schemaId);
      commonToIntermediateIndexes.add(getIndexTranslation(commonSchema, sourceSchema));
      groupToIntermediateIndexes.add(getIndexTranslation(groupSchema, sourceSchema));
      if(specificSchemas != null && !specificSchemas.isEmpty()) {
        Schema particularSchema = specificSchemas.get(schemaId);
        specificToIntermediateIndexes.add(getIndexTranslation(particularSchema,
            sourceSchema));
      }
    }
    commonToIntermediateIndexes = Collections

View Full Code Here

      throw new TupleMRException("Not able to define field aliases with just one schema");
    }
    failIfNull(schemaName,"Need to specify schema");
    failIfEmpty(aliases.getAliases().entrySet(),"Aliases empty");
    
    Schema schema = getSchemaByName(schemaName);
    if (schema == null){
      throw new TupleMRException("Unknown schema : " +schemaName);
    }
    
    if (this.fieldAliases.get(schemaName) != null){
      throw new TupleMRException("Already aliases set for schema '" + schemaName +"'");
    }
    
    for (Map.Entry<String,String> entry : aliases.getAliases().entrySet()){
      String alias = entry.getKey();
      if (schema.containsField(alias)){
        throw new TupleMRException("Forbidden alias '" + alias + 
            "'. Schema '" + schema + "' already contains a field with that name");
      }
      
      String referenced = entry.getValue();
      if (!schema.containsField(referenced)){
        throw new TupleMRException("Incorrect alias.Schema '" + schemaName +
            "' doesn't contain field: '" + referenced);
      }
    }
    this.fieldAliases.put(schemaName,aliases.getAliases());

View Full Code Here

      if(ordering.getSchemaOrderIndex() != null) {
        throw new TupleMRException(
            "Not able to use source order when just one source specified");
      }
    }
    Schema firstSchema = schemas.get(0);
    
    for(SortElement sortElement : ordering.getElements()) {
      if(!fieldPresentInAllSchemas(sortElement.getName())) {
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' . Not present in all sources");
      }
      if(!fieldSameTypeInAllSources(sortElement.getName())) {
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' since its type differs among sources");
      }
      
      if (sortElement.getCustomComparator() != null){
        Field field = firstSchema.getField(sortElement.getName());
        if (field.getType() != Type.OBJECT){
          throw new TupleMRException("Not allowed to specify custom comparator for type="+field.getType());
        }
      }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.datasalt.pangool.io.Schema$Field$Props

aleGrammar.ALEParser.ExtendedVertex

com.datasalt.pangool.examples.avro.AvroTopicalWordCount

com.datasalt.pangool.examples.avro.AvroTweetsJoin

com.datasalt.pangool.examples.gameoflife.GameOfLifeJob

com.datasalt.pangool.examples.movingaverage.MovingAverage

com.datasalt.pangool.examples.secondarysort.SecondarySort

com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort

com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount

com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint

com.datasalt.pangool.examples.topnhashtags.TopNHashTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.