Examples of com.datasalt.pangool.io.Schema

com.datasalt.pangool.io.Schema
A list of {@link Field} elements that a {@link ITuple} instance contains.

          "Need to specify source order in common OrderBy when using specific OrderBy");
    }
    if(ordering.getSchemaOrderIndex() != null) {
      throw new TupleMRException("Not allowed to set source order in specific order");
    }
    Schema schema = getSchemaByName(schemaName);
    Map<String,String> aliases = fieldAliases.get(schema.getName());
    for(SortElement e : ordering.getElements()) {
      if(!Schema.containsFieldUsingAlias(schema,e.getName(), aliases)){
        throw new TupleMRException("Source '" + schemaName + "' doesn't contain field '"
            + e.getName());
      }
      if (e.getCustomComparator() != null){
        Field field = schema.getField(e.getName());
        if (field == null){
            field = schema.getField(aliases.get(e.getName()));
        }
        if (field.getType() != Type.OBJECT){
          throw new TupleMRException("Not allowed to set custom comparator for type="+field.getType());
        }
      }

View Full Code Here

    ITuple tuple = reuse;
    if (tuple == null){
      tuple = new Tuple(pangoolSchema);
    }
    
    Schema pangoolSchema = tuple.getSchema();
    for(org.apache.avro.Schema.Field avroField : avroSchema.getFields()) {
      int pos = avroField.pos();
      Object objRecord = record.get(pos);
      Field pangoolField = pangoolSchema.getField(pos);
      switch(pangoolField.getType()){
      case INT:
      case LONG:
      case BOOLEAN:
      case FLOAT:

View Full Code Here

    }
  }


  protected int compareMultipleSources(byte[] b1, int s1, int l1, byte[] b2, int s2,
      int l2) throws IOException {
    Schema commonSchema = serInfo.getCommonSchema();
    Criteria commonOrder = tupleMRConf.getCommonCriteria();


    int comparison = compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets);
    if(comparison != 0) {
      return comparison;
    }


    int schemaId1 = readVInt(b1, offsets.offset1);
    int schemaId2 = readVInt(b2, offsets.offset2);
    if(schemaId1 != schemaId2) {
      int r = schemaId1 - schemaId2;
      return (tupleMRConf.getSchemasOrder() == Order.ASC) ? r : -r;
    }


    int vintSize = WritableUtils.decodeVIntSize(b1[offsets.offset1]);
    offsets.offset1 += vintSize;
    offsets.offset2 += vintSize;


    // sources are the same
    Criteria criteria = tupleMRConf.getSpecificOrderBys().get(schemaId1);
    if(criteria == null) {
      return 0;
    }


    Schema specificSchema = serInfo.getSpecificSchema(schemaId1);
    return compare(b1, offsets.offset1, b2, offsets.offset2, specificSchema, criteria,
        offsets);


  }

View Full Code Here


  }


  private int compareOneSource(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2)
      throws IOException {
    Schema commonSchema = serInfo.getCommonSchema();
    Criteria commonOrder = tupleMRConf.getCommonCriteria();
    return compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets);
  }

View Full Code Here

        }
      }
      fields.add(pangoolField);
      
    }
    Schema schema = new Schema(avroSchema.getFullName(), fields);
    return schema;
  }

View Full Code Here

  }


  @Override
  public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
    try{
    Schema groupSchema = serInfo.getGroupSchema();
    return compare(b1,s1,b2,s2,groupSchema,groupCriteria,offsets);
    } catch(IOException e){
      throw new RuntimeException(e);
    }
  }

View Full Code Here


  @Override
  public RecordWriter<ITuple, NullWritable> getRecordWriter(TaskAttemptContext context)
      throws IOException, InterruptedException {


    Schema pangoolOutputSchema = Schema.parse(this.pangoolOutputSchema);
    org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(pangoolOutputSchema);
    DataFileWriter<Record> writer = new DataFileWriter<Record>(
        new ReflectDatumWriter<Record>());


    // Compression etc - use Avro codecs

View Full Code Here

  static Schema getURLRegisterSchema() {
    List<Field> urlRegisterFields = new ArrayList<Field>();
    urlRegisterFields.add(Field.create("url",Type.STRING));
    urlRegisterFields.add(Field.create("timestamp",Type.LONG));
    urlRegisterFields.add(Field.create("ip",Type.STRING));
    return new Schema("urlRegister", urlRegisterFields);    
  }

View Full Code Here


  static Schema getURLMapSchema() {
    List<Field> urlMapFields = new ArrayList<Field>();
    urlMapFields.add(Field.create("url",Type.STRING));
    urlMapFields.add(Field.create("canonicalUrl",Type.STRING));
    return new Schema("urlMap", urlMapFields);
  }

View Full Code Here

    // Serialize common
    tupleSerializer.write(commonSchema, tuple, commonTranslation, serInfo.getCommonSchemaSerializers());
    // Serialize source id
    WritableUtils.writeVInt(tupleSerializer.getOut(), schemaId);
    // Serialize rest of the fields
    Schema specificSchema = serInfo.getSpecificSchema(schemaId);
    int[] specificTranslation = serInfo
        .getSpecificSchemaIndexTranslation(schemaId);
    tupleSerializer.write(specificSchema, tuple, specificTranslation, serInfo.getSpecificSchemaSerializers().get(schemaId));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.datasalt.pangool.io.Schema

com.datasalt.pangool.examples.avro.AvroCustomSerializationJob

com.datasalt.pangool.examples.avro.AvroTopicalWordCount

com.datasalt.pangool.examples.avro.AvroTweetsJoin

com.datasalt.pangool.examples.gameoflife.GameOfLifeJob

com.datasalt.pangool.examples.movingaverage.MovingAverage

com.datasalt.pangool.examples.secondarysort.SecondarySort

com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort

com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount

com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint

com.datasalt.pangool.examples.topnhashtags.TopNHashTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.