Package com.datasalt.pangool.io

Examples of com.datasalt.pangool.io.Schema

90
91
92
93
94
95
96
97
98
99
    // serialize common
    write(commonSchema, tuple, commonTranslation, out,serInfo.getCommonSchemaSerializers());
    // serialize source id
    WritableUtils.writeVInt(out, schemaId);
    // serialize rest of the fields
    Schema specificSchema = serInfo.getSpecificSchema(schemaId);
    int[] specificTranslation = serInfo
        .getSpecificSchemaIndexTranslation(schemaId);
    write(specificSchema, tuple, specificTranslation, out,serInfo.getSpecificSchemaSerializers().get(schemaId));
  }
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    return reuse;
  }

  public void readFields(ITuple tuple, DataInputStream input,FieldDeserializer[] customDeserializers)
      throws IOException {
    Schema schema = tuple.getSchema();
    for(int index = 0; index < schema.getFields().size(); index++) {
      FieldDeserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);
      switch(field.getType()){
      case INT:  tuple.set(index,WritableUtils.readVInt(input)); break;
      case LONG: tuple.set(index,WritableUtils.readVLong(input)); break;
      case DOUBLE: tuple.set(index,input.readDouble()); break;
      case FLOAT: tuple.set(index,input.readFloat()); break;
View Full Code Here
140
141
142
143
144
145
146
  }

  private void calculateGroupSchema() {
    List<Field> fields = commonSchema.getFields();
    List<Field> groupFields = fields.subList(0, mrConfig.getGroupByFields().size());
    this.groupSchema = new Schema("group", groupFields);
  }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
 
  private void initSpecificSchemaSerialization(){
    specificSerializers = new ArrayList<FieldSerializer[]>();
    specificDeserializers = new ArrayList<FieldDeserializer[]>();
    for(int i= 0 ; i < specificSchemas.size(); i++){
      Schema specificSchema = specificSchemas.get(i);
      specificSerializers.add(getSerializers(specificSchema));
      specificDeserializers.add(getDeserializers(specificSchema));
    }
  }
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
      fieldsToPartition.add(posFields);
    }
  }

  private void calculateOneIntermediateCommonSchema() throws TupleMRException {
    Schema intermediateSchema = mrConfig.getIntermediateSchemas().get(0);
    Criteria commonSortCriteria = mrConfig.getCommonCriteria();
    List<Field> commonFields = new ArrayList<Field>();
    for(SortElement sortElement : commonSortCriteria.getElements()) {
      String fieldName = sortElement.getName();
      Field field = checkFieldInAllSchemas(fieldName);
      commonFields.add(Field.cloneField(field, fieldName));
    }

    // adding the rest
    for(Field field : intermediateSchema.getFields()) {
      Map<String,String> aliases = mrConfig.getFieldAliases(intermediateSchema.getName());
      if(!containsField(field.getName(),commonFields,aliases))  {
          commonFields.add(field);
      }
    }
    this.commonSchema = new Schema("common", commonFields);
  }
View Full Code Here
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
      Field field = checkFieldInAllSchemas(fieldName);
     
      commonFields.add(Field.cloneField(field,fieldName));
    }

    this.commonSchema = new Schema("common", commonFields);
    this.specificSchemas = new ArrayList<Schema>();
    List<List<Field>> specificFieldsBySource = new ArrayList<List<Field>>();

    for(int schemaId = 0; schemaId < mrConfig.getNumIntermediateSchemas(); schemaId++) {
      Criteria specificCriteria = mrConfig.getSpecificOrderBys().get(schemaId);
      List<Field> specificFields = new ArrayList<Field>();
      if(specificCriteria != null) {
        for(SortElement sortElement : specificCriteria.getElements()) {
          String fieldName = sortElement.getName();
          Field field = checkFieldInSchema(fieldName, schemaId);
          specificFields.add(Field.cloneField(field,fieldName));
        }
      }
      specificFieldsBySource.add(specificFields);
    }

    for(int i = 0; i < mrConfig.getNumIntermediateSchemas(); i++) {
      Schema sourceSchema = mrConfig.getIntermediateSchema(i);
      List<Field> specificFields = specificFieldsBySource.get(i);
      for(Field field : sourceSchema.getFields()) {
        Map<String,String> sourceAliases = mrConfig.getFieldAliases(sourceSchema.getName());
        if(!containsField(field.getName(),commonSchema.getFields(),sourceAliases)
            && !containsField(field.getName(), specificFields,sourceAliases)) {
          specificFields.add(field);
        }
      }
      this.specificSchemas.add(new Schema("specific", specificFields));
    }
    this.specificSchemas = Collections.unmodifiableList(this.specificSchemas);
  }
View Full Code Here
326
327
328
329
330
331
332
333
334
335
336
337
338
    return field;
  }

  private Field checkFieldInSchema(String fieldName, int schemaId)
      throws TupleMRException {
    Schema schema = mrConfig.getIntermediateSchema(schemaId);
    Field field = getFieldUsingAliases(schema,fieldName);
    if(field == null) {
      throw new TupleMRException("Field '" + fieldName + "' not present in source '"
          + schema.getName() + "' " + schema);
    }
    return field;
  }
View Full Code Here
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
    return groupSchema;
  }

  private void calculateIndexTranslations() {
    for(int schemaId = 0; schemaId < mrConfig.getIntermediateSchemas().size(); schemaId++) {
      Schema sourceSchema = mrConfig.getIntermediateSchema(schemaId);
      commonToIntermediateIndexes.add(getIndexTranslation(commonSchema, sourceSchema));
      groupToIntermediateIndexes.add(getIndexTranslation(groupSchema, sourceSchema));
      if(specificSchemas != null && !specificSchemas.isEmpty()) {
        Schema particularSchema = specificSchemas.get(schemaId);
        specificToIntermediateIndexes.add(getIndexTranslation(particularSchema,
            sourceSchema));
      }
    }
    commonToIntermediateIndexes = Collections
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
      throw new TupleMRException("Not able to define field aliases with just one schema");
    }
    failIfNull(schemaName,"Need to specify schema");
    failIfEmpty(aliases.getAliases().entrySet(),"Aliases empty");
   
    Schema schema = getSchemaByName(schemaName);
    if (schema == null){
      throw new TupleMRException("Unknown schema : " +schemaName);
    }
   
    if (this.fieldAliases.get(schemaName) != null){
      throw new TupleMRException("Already aliases set for schema '" + schemaName +"'");
    }
   
    for (Map.Entry<String,String> entry : aliases.getAliases().entrySet()){
      String alias = entry.getKey();
      if (schema.containsField(alias)){
        throw new TupleMRException("Forbidden alias '" + alias +
            "'. Schema '" + schema + "' already contains a field with that name");
      }
     
      String referenced = entry.getValue();
      if (!schema.containsField(referenced)){
        throw new TupleMRException("Incorrect alias.Schema '" + schemaName +
            "' doesn't contain field: '" + referenced);
      }
    }
    this.fieldAliases.put(schemaName,aliases.getAliases());
View Full Code Here
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
      if(ordering.getSchemaOrderIndex() != null) {
        throw new TupleMRException(
            "Not able to use source order when just one source specified");
      }
    }
    Schema firstSchema = schemas.get(0);
   
    for(SortElement sortElement : ordering.getElements()) {
      if(!fieldPresentInAllSchemas(sortElement.getName())) {
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' . Not present in all sources");
      }
      if(!fieldSameTypeInAllSources(sortElement.getName())) {
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' since its type differs among sources");
      }
     
      if (sortElement.getCustomComparator() != null){
        Field field = firstSchema.getField(sortElement.getName());
        if (field.getType() != Type.OBJECT){
          throw new TupleMRException("Not allowed to specify custom comparator for type="+field.getType());
        }
      }
    }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.Schema

  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin
  • com.datasalt.pangool.examples.gameoflife.GameOfLifeJob
  • com.datasalt.pangool.examples.movingaverage.MovingAverage
  • com.datasalt.pangool.examples.secondarysort.SecondarySort
  • com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort
  • com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount
  • com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint
  • com.datasalt.pangool.examples.topnhashtags.TopNHashTags
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.