Examples of com.datasalt.pangool.io.Schema

com.datasalt.pangool.io.Schema
A list of {@link Field} elements that a {@link ITuple} instance contains.

      throw new TupleMRException("Not able to define field aliases with just one schema");
    }
    failIfNull(schemaName, "Need to specify schema");
    failIfEmpty(aliases.getAliases().entrySet(), "Aliases empty");


    Schema schema = getSchemaByName(schemaName);
    if (schema == null) {
      throw new TupleMRException("Unknown schema : " + schemaName);
    }


    if (this.fieldAliases.get(schemaName) != null) {
      throw new TupleMRException("Already aliases set for schema '" + schemaName + "'");
    }


    for (Map.Entry<String, String> entry : aliases.getAliases().entrySet()) {
      String alias = entry.getKey();
      if (schema.containsField(alias)) {
        throw new TupleMRException("Forbidden alias '" + alias +
            "'. Schema '" + schema + "' already contains a field with that name");
      }


      String referenced = entry.getValue();
      if (!schema.containsField(referenced)) {
        throw new TupleMRException("Incorrect alias.Schema '" + schemaName +
            "' doesn't contain field: '" + referenced);
      }
    }
    this.fieldAliases.put(schemaName, aliases.getAliases());

View Full Code Here

      if (ordering.getSchemaOrderIndex() != null) {
        throw new TupleMRException(
            "Not able to use source order when just one source specified");
      }
    }
    Schema firstSchema = schemas.get(0);


    for (SortElement sortElement : ordering.getElements()) {
      if (!fieldPresentInAllSchemas(sortElement.getName())) {
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' . Not present in all sources");
      }
      if (!fieldSameTypeInAllSources(sortElement.getName())) {
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' since its type differs among sources");
      }


      if (sortElement.getCustomComparator() != null) {
        Field field = firstSchema.getField(sortElement.getName());
        if (field.getType() != Type.OBJECT) {
          throw new TupleMRException("Not allowed to specify custom comparator for type=" + field.getType());
        }
      }
    }

View Full Code Here

          "Need to specify source order in common OrderBy when using specific OrderBy");
    }
    if (ordering.getSchemaOrderIndex() != null) {
      throw new TupleMRException("Not allowed to set source order in specific order");
    }
    Schema schema = getSchemaByName(schemaName);
    Map<String, String> aliases = fieldAliases.get(schema.getName());
    for (SortElement e : ordering.getElements()) {
      if (!Schema.containsFieldUsingAlias(schema, e.getName(), aliases)) {
        throw new TupleMRException("Source '" + schemaName + "' doesn't contain field '"
            + e.getName());
      }
      if (e.getCustomComparator() != null) {
        Field field = schema.getField(e.getName());
        if (field == null) {
          field = schema.getField(aliases.get(e.getName()));
        }
        if (field.getType() != Type.OBJECT) {
          throw new TupleMRException("Not allowed to set custom comparator for type=" + field.getType());
        }
      }

View Full Code Here

    }
  }


  protected int compareMultipleSources(byte[] b1, int s1, int l1, byte[] b2, int s2,
                                       int l2) throws IOException {
    Schema commonSchema = serInfo.getCommonSchema();
    Criteria commonOrder = tupleMRConf.getCommonCriteria();


    int comparison = compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets, nulls);
    if (comparison != 0) {
      return comparison;
    }


    int schemaId1 = readVInt(b1, offsets.offset1);
    int schemaId2 = readVInt(b2, offsets.offset2);
    if (schemaId1 != schemaId2) {
      int r = schemaId1 - schemaId2;
      return (tupleMRConf.getSchemasOrder() == Order.ASC) ? r : -r;
    }


    int vintSize = WritableUtils.decodeVIntSize(b1[offsets.offset1]);
    offsets.offset1 += vintSize;
    offsets.offset2 += vintSize;


    // sources are the same
    Criteria criteria = tupleMRConf.getSpecificOrderBys().get(schemaId1);
    if (criteria == null) {
      return 0;
    }


    Schema specificSchema = serInfo.getSpecificSchema(schemaId1);
    return compare(b1, offsets.offset1, b2, offsets.offset2, specificSchema, criteria,
        offsets, nulls);


  }

View Full Code Here


  }


  private int compareOneSource(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2)
      throws IOException {
    Schema commonSchema = serInfo.getCommonSchema();
    Criteria commonOrder = tupleMRConf.getCommonCriteria();
    return compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets, nulls);
  }

View Full Code Here

    }
  }


  @Override
  public void serialize(ITuple tuple) throws IOException {
    Schema schema = (this.schema != null) ? this.schema : tuple.getSchema();
    write(schema, tuple, null, customSerializers);
  }

View Full Code Here

  }


  @Override
  public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
    try{
    Schema groupSchema = serInfo.getGroupSchema();
    return compare(b1,s1,b2,s2,groupSchema,groupCriteria,offsets,nulls);
    } catch(IOException e){
      throw new RuntimeException(e);
    }
  }

View Full Code Here

    ITuple tuple = reuse;
    if(tuple == null) {
      tuple = new Tuple(pangoolSchema);
    }


    Schema pangoolSchema = tuple.getSchema();
    for(org.apache.avro.Schema.Field avroField : avroSchema.getFields()) {
      int pos = avroField.pos();
      Object objRecord = record.get(pos);
      Field pangoolField = pangoolSchema.getField(pos);
      switch(pangoolField.getType()) {
      case INT:
      case LONG:
      case BOOLEAN:
      case FLOAT:

View Full Code Here

    }
  }


  protected int compareMultipleSources(byte[] b1, int s1, int l1, byte[] b2, int s2,
      int l2) throws IOException {
    Schema commonSchema = serInfo.getCommonSchema();
    Criteria commonOrder = tupleMRConf.getCommonCriteria();


    int comparison = compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets);
    if(comparison != 0) {
      return comparison;
    }


    int schemaId1 = readVInt(b1, offsets.offset1);
    int schemaId2 = readVInt(b2, offsets.offset2);
    if(schemaId1 != schemaId2) {
      int r = schemaId1 - schemaId2;
      return (tupleMRConf.getSchemasOrder() == Order.ASC) ? r : -r;
    }


    int vintSize = WritableUtils.decodeVIntSize(b1[offsets.offset1]);
    offsets.offset1 += vintSize;
    offsets.offset2 += vintSize;


    // sources are the same
    Criteria criteria = tupleMRConf.getSpecificOrderBys().get(schemaId1);
    if(criteria == null) {
      return 0;
    }


    Schema specificSchema = serInfo.getSpecificSchema(schemaId1);
    return compare(b1, offsets.offset1, b2, offsets.offset2, specificSchema, criteria,
        offsets);


  }

View Full Code Here


  }


  private int compareOneSource(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2)
      throws IOException {
    Schema commonSchema = serInfo.getCommonSchema();
    Criteria commonOrder = tupleMRConf.getCommonCriteria();
    return compare(b1, s1, b2, s2, commonSchema, commonOrder, offsets);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.datasalt.pangool.io.Schema

com.datasalt.pangool.examples.avro.AvroCustomSerializationJob

com.datasalt.pangool.examples.avro.AvroTopicalWordCount

com.datasalt.pangool.examples.avro.AvroTweetsJoin

com.datasalt.pangool.examples.gameoflife.GameOfLifeJob

com.datasalt.pangool.examples.movingaverage.MovingAverage

com.datasalt.pangool.examples.secondarysort.SecondarySort

com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort

com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount

com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint

com.datasalt.pangool.examples.topnhashtags.TopNHashTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.