Examples of com.linkedin.data.schema.SchemaParser

com.linkedin.data.schema.SchemaParser
Schema Parser.
Inspired by Avro 1.4.1 specification.
The parser can use an optional {@link DataSchemaResolver} to resolve namesnot found parsed from its input.
@author slim

    testInputs(inputs, true);
  }


  private static NamedDataSchema dataSchemaFromString(String s, boolean isAvroUnionMode) throws IOException
  {
    SchemaParser parser = new SchemaParser();
    parser.getValidationOptions().setAvroUnionMode(isAvroUnionMode);
    parser.parse(TestUtil.inputStreamFromString(s));
    if (parser.hasError())
    {
      TestUtil.out.println("ERROR: " + parser.errorMessage());
      return null;
    }
    return (NamedDataSchema) parser.topLevelDataSchemas().get(parser.topLevelDataSchemas().size() - 1);
  }

View Full Code Here

      Predicate predicate = (Predicate) row[1];
      String expected = (String) row[2];
      NamedDataSchema schema = dataSchemaFromString(schemaText, isAvroUnionMode);


      DataSchema filteredSchema = null;
      SchemaParser parser = new SchemaParser();
      parser.getValidationOptions().setAvroUnionMode(isAvroUnionMode);
      filteredSchema = Filters.removeByPredicate(schema, predicate, parser);
      if (filteredSchema != null)
      {
        // Schema string match
        String expectedSchemaText = expected;
        DataSchema expectedSchema = dataSchemaFromString(expectedSchemaText, isAvroUnionMode);
        assertEquals(filteredSchema.toString(), expectedSchema.toString());
        assertEquals(filteredSchema, expectedSchema);
      }
      else
      {
        String parserMessage = parser.errorMessage();
        assertTrue(parserMessage.contains(expected), "\nContains :" + expected + "\nActual   :" + parserMessage);
      }
    }
  }

View Full Code Here

    for (String good : goodInputs)
    {
      NamedDataSchema dataSchema = (NamedDataSchema) TestUtil.dataSchemaFromString(good);


      DataMap mapFromString = TestUtil.dataMapFromString(good);
      SchemaParser parser = new SchemaParser();
      DataSchema schemaFromMap = Conversions.dataMapToDataSchema(mapFromString, parser);


      assertEquals(schemaFromMap, dataSchema);
    }


    for (String bad : badInputs)
    {
      DataMap mapFromString = TestUtil.dataMapFromString(bad);
      SchemaParser parser = new SchemaParser();
      DataSchema schemaFromMap = Conversions.dataMapToDataSchema(mapFromString, parser);
      assertNull(schemaFromMap);
      assertTrue(parser.hasError());
    }
  }

View Full Code Here

   * @return the {@link NamedDataSchema} is found in the input stream, else return null.
   */
  protected NamedDataSchema parse(InputStream inputStream, final DataSchemaLocation location, String name, StringBuilder errorMessageBuilder)
  {
    NamedDataSchema schema = null;
    SchemaParser parser = _parserFactory.create(this);
    parser.setLocation(location);
    //out.println("start parsing " + location);


    parser.parse(new FilterInputStream(inputStream)
    {
      @Override
      public String toString()
      {
        return location.toString();
      }
    });


    if (parser.hasError())
    {
      //out.println(parser.errorMessageBuilder().toString());


      errorMessageBuilder.append("Error parsing ").append(location).append(" for \"").append(name).append("\".\n");
      errorMessageBuilder.append(parser.errorMessageBuilder());
      errorMessageBuilder.append("Done parsing ").append(location).append(".\n");


      _badLocations.add(location);
    }
    else

View Full Code Here

   * @return the data schemas within the source file.
   * @throws IOException if there is a file access error.
   */
  protected List<DataSchema> parseSchema(final File schemaSourceFile) throws IOException
  {
    SchemaParser parser = new SchemaParser(getSchemaResolver());
    FileInputStream schemaStream = new SchemaFileInputStream(schemaSourceFile);
    try
    {
      parser.setLocation(new FileDataSchemaLocation(schemaSourceFile));
      parser.parse(schemaStream);
      if (parser.hasError())
      {
        return Collections.emptyList();
      }
      return parser.topLevelDataSchemas();
    }
    finally
    {
      schemaStream.close();
      if (parser.hasError())
      {
        getMessage().append(schemaSourceFile.getPath() + ",");
        getMessage().append(parser.errorMessage());
      }
    }
  }

View Full Code Here

   * @throws IllegalArgumentException if the Avro {@link Schema} cannot be translated.
   */
  public static DataSchema avroToDataSchema(String avroSchemaInJson, AvroToDataSchemaTranslationOptions options)
    throws IllegalArgumentException
  {
    SchemaParser parser = SchemaParserFactory.instance().create(null);
    parser.getValidationOptions().setAvroUnionMode(true);
    parser.parse(avroSchemaInJson);
    if (parser.hasError())
    {
      throw new IllegalArgumentException(parser.errorMessage());
    }
    assert(parser.topLevelDataSchemas().size() == 1);
    DataSchema dataSchema = parser.topLevelDataSchemas().get(0);
    DataSchema resultDataSchema = null;


    AvroToDataSchemaTranslationMode translationMode = options.getTranslationMode();
    if (translationMode == AvroToDataSchemaTranslationMode.RETURN_EMBEDDED_SCHEMA ||
        translationMode == AvroToDataSchemaTranslationMode.VERIFY_EMBEDDED_SCHEMA)
    {
      // check for embedded schema


      Object dataProperty = dataSchema.getProperties().get(SchemaTranslator.DATA_PROPERTY);
      if (dataProperty != null && dataProperty.getClass() == DataMap.class)
      {
        Object schemaProperty = ((DataMap) dataProperty).get(SchemaTranslator.SCHEMA_PROPERTY);
        if (schemaProperty.getClass() == DataMap.class)
        {
          SchemaParser embeddedSchemaParser = SchemaParserFactory.instance().create(null);
          embeddedSchemaParser.parse(Arrays.asList(schemaProperty));
          if (embeddedSchemaParser.hasError())
          {
            throw new IllegalArgumentException("Embedded schema is invalid\n" + embeddedSchemaParser.errorMessage());
          }
          assert(embeddedSchemaParser.topLevelDataSchemas().size() == 1);
          resultDataSchema = embeddedSchemaParser.topLevelDataSchemas().get(0);


          if (translationMode == AvroToDataSchemaTranslationMode.VERIFY_EMBEDDED_SCHEMA)
          {
            // additional verification to make sure that embedded schema translates to Avro schema
            DataToAvroSchemaTranslationOptions dataToAvdoSchemaOptions = new DataToAvroSchemaTranslationOptions();

View Full Code Here

      avroSchemaFieldsJson
        .replace("##ANYRECORD_FULL_JSON", ANYRECORD_AVRO_SCHEMA_JSON)
        .replace("##ANYRECORD_NAME", ANYRECORD_AVRO_FULL_NAME);
    String fullAvroSchemaJson = AVRO_SCHEMA_JSON_TEMPLATE.replace("##FIELDS", avroSchemaFieldsJsonAfterVariableExpansion);


    SchemaParser parser = TestUtil.schemaParserFromString(fullSchemaJson);
    assertFalse(parser.hasError(), parser.errorMessage());
    RecordDataSchema schema = (RecordDataSchema) parser.topLevelDataSchemas().get(2);


    String avroJsonOutput = SchemaTranslator.dataToAvroSchemaJson(schema);
    assertEquals(TestUtil.dataMapFromString(avroJsonOutput), TestUtil.dataMapFromString(fullAvroSchemaJson));
    Schema avroSchema = Schema.parse(avroJsonOutput);
    Schema avroSchema2 = SchemaTranslator.dataToAvroSchema(schema);

View Full Code Here

      String schemaText = (String) row[i++];
      Predicate predicate = (Predicate) row[i++];
      String avroSchemaText = (String) row[i++];


      RecordDataSchema schema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText);
      NamedDataSchema filteredSchema = Filters.removeByPredicate(schema, predicate, new SchemaParser());
      Schema filteredAvroSchema = SchemaTranslator.dataToAvroSchema(filteredSchema);


      Schema expectedAvroSchema = Schema.parse(avroSchemaText);
      assertEquals(filteredAvroSchema, expectedAvroSchema);

View Full Code Here

      String schemaText = (String) row[i++];


      Schema avroSchema = Schema.parse(avroSchemaText);
      System.out.println(avroSchema);
      RecordDataSchema schema = (RecordDataSchema) SchemaTranslator.avroToDataSchema(avroSchema);
      RecordDataSchema filteredSchema = (RecordDataSchema) Filters.removeByPredicate(schema, predicate, new SchemaParser());


      DataSchema expectedSchema = TestUtil.dataSchemaFromString(schemaText);
      System.out.println(filteredSchema);
      assertEquals(filteredSchema, expectedSchema);

View Full Code Here

              Object schemaProperty = ((DataMap) dataProperty).get(SchemaTranslator.SCHEMA_PROPERTY);
              assertNotNull(schemaProperty);
              assertTrue(schemaProperty instanceof DataMap);


              // make sure embedded schema is same as the original schema
              SchemaParser schemaParser = TestUtil.schemaParserFromObjects(Arrays.asList(schemaProperty));
              DataSchema embeddedSchema = schemaParser.topLevelDataSchemas().get(0);
              assertEquals(embeddedSchema, schema.getDereferencedDataSchema());


              // look for optional default mode
              Object optionalDefaultModeProperty = ((DataMap) dataProperty).get(SchemaTranslator.OPTIONAL_DEFAULT_MODE_PROPERTY);
              assertNotNull(optionalDefaultMode);
              assertEquals(optionalDefaultModeProperty, optionalDefaultMode.toString());
            }
            else
            {
              // embeddedSchema is not enabled or
              // for unions and primitives, we never embed the pegasus schema
              if (embedSchemaMode == EmbedSchemaMode.NONE && hasEmbeddedSchema(schema))
              {
                // make sure no embedded schema when
                DataMap resultAvroDataMap = TestUtil.dataMapFromString(avroTextFromSchema);
                assertFalse(resultAvroDataMap.containsKey(SchemaTranslator.DATA_PROPERTY));
              }
              assertEquals(avroTextFromSchema, expectedAvroText);
            }


            String postTranslateSchemaText = schema.toString();
            assertEquals(preTranslateSchemaText, postTranslateSchemaText);


            // make sure Avro accepts it
            Schema avroSchema = Schema.parse(avroTextFromSchema);
            if (debug) System.out.println("AvroSchema: " + avroSchema);


            SchemaParser parser = new SchemaParser();
            ValidationOptions options = new ValidationOptions();
            options.setAvroUnionMode(true);
            parser.setValidationOptions(options);
            parser.parse(avroTextFromSchema);
            assertFalse(parser.hasError(), parser.errorMessage());


            if (optionalDefaultMode == DataToAvroSchemaTranslationOptions.DEFAULT_OPTIONAL_DEFAULT_MODE)
            {
              // use other dataToAvroSchemaJson
              String avroSchema2Json = SchemaTranslator.dataToAvroSchemaJson(

View Full Code Here

0 1

TOP

Related Classes of com.linkedin.data.schema.SchemaParser

com.linkedin.data.avro.SchemaTranslator

com.linkedin.data.avro.TestCustomAvroSchema

com.linkedin.data.avro.TestFilteredSchemaDataTranslation

com.linkedin.data.avro.TestSchemaTranslator

com.linkedin.data.codec.DataLocation

com.linkedin.data.DataList

com.linkedin.data.DataMap

com.linkedin.data.schema.generator.AbstractGenerator

com.linkedin.data.schema.generator.SchemaSampleDataGenerator

com.linkedin.data.schema.resolver.AbstractDataSchemaResolver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.