Package com.odiago.flumebase.parser

Examples of com.odiago.flumebase.parser.TypedField

48
49
50
51
52
53
54
55
56
57
58
59
60
61
  @Override
  public void takeEvent(EventWrapper e) throws IOException, InterruptedException {
    GenericData.Record record = new GenericData.Record(getOutputSchema());

    for (int i = 0; i < mInputFields.size(); i++) {
      TypedField inField = mInputFields.get(i);
      TypedField outField = mOutputFields.get(i);

      String outFieldName = outField.getAvroName();
      record.put(outFieldName, nativeToAvro(e.getField(inField), outField.getType()));
    }

    emitAvroRecord(record, e.getEvent());
  }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
        return false;
      }
      List<TypedField> columnFields = streamSym.getFields();

      for (int i = 0; i < columnFields.size(); i++) {
        TypedField col = columnFields.get(i);
        Type colType = col.getType();
       
        // Get the schema field that matches this column name.
        Schema.Field schemaField = null;
        for (Schema.Field testSchemaField : schemaFields) {
          if (testSchemaField.name().equals(col.getUserAlias())) {
            schemaField = testSchemaField;
            break;
          }
        }
       
        if (null == schemaField) {
          // Can't find a field in the schema to match the current column.
          LOG.error("The Avro schema does not contain a field with the same name "
              + "as column '" + col.getUserAlias() + "'.");
          return false;
        }
       
        // Are the schemas compatible?
        if (!schemaField.schema().equals(colType.getAvroSchema())) {
          boolean warned = false;

          if (colType.isNullable() && colType.isPrimitive()) {
            // A common error is that the rtsql type is nullable and the schema
            // type isn't. Give a specific suggestion in this case.
            Type nonNullVersion = Type.getPrimitive(colType.getPrimitiveTypeName());
            if (schemaField.schema().equals(nonNullVersion.getAvroSchema())) {
              LOG.error("Column " + col.getUserAlias() + " has type " + colType
                  + ", but requires type " + nonNullVersion + " to match the Avro schema.");
              warned = true;
            }
          }

          if (!warned) {
            // Give a generic error message.
            LOG.error("Column " + col.getUserAlias() + " has type " + colType
                + " with avro schema " + colType.getAvroSchema()
                + " but the stream schema field " + schemaField.name() + " has "
                + " an incompatible Avro schema: " + schemaField.schema());
          }
          return false;
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
      sbStringCol.append("a");
    }

    String stringCol = sbStringCol.toString();

    streamBuilder.addField(new TypedField("a", Type.getPrimitive(Type.TypeName.INT)));
    streamBuilder.addField(new TypedField("b", Type.getNullable(Type.TypeName.INT)));
    streamBuilder.addField(new TypedField("c", Type.getNullable(Type.TypeName.STRING)));
    for (int i = 0; i < NUM_RECORDS; i++) {
      StringBuilder sb = new StringBuilder();
      sb.append(i);
      sb.append(",");
      int j = 10 * i;
 
View Full Code Here
96
97
98
99
100
101
102
  public void addField(TypedField tf) {
    mFields.add(tf);
  }

  public void addField(String fieldName, Type fieldType) {
    addField(new TypedField(fieldName, fieldType));
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
   */
  private class EventGenThread extends Thread {
    public void run() {
      // If the user has specified a column to extract the timestamp from, get it here.
      String timestampCol = mStream.getFormatSpec().getParam(TIMESTAMP_COL_KEY);
      TypedField timestampField = null;
      if (null != timestampCol) {
        // timestampCol refers to a user-selected name for the column. Translate that
        // to the internal ("avro") name for the column.
        for (TypedField field : mFields) {
          if (field.getUserAlias().equals(timestampCol)) {
            timestampField = field;
            break;
          }
        }

        if (null == timestampField) {
          LOG.warn("Could not find column '" + timestampCol + "' to use for timestamps.");
          LOG.warn("Timestamps will be generated based on the local system clock.");
        } else if (!timestampField.getType().getPrimitiveTypeName()
            .equals(Type.TypeName.TIMESTAMP)) {
          LOG.warn("Specified timestamp.col '" + timestampCol + "' has type "
              + timestampField.getType() + ", but we need TIMESTAMP.");
          LOG.warn("Timestamps will be generated based on the local system clock.");
          timestampField = null;
        } else {
          // Ensure that we normalize the type associated with this column for ts retrieval.
          timestampField = new TypedField(timestampField.getAvroName(),
              Type.getNullable(Type.TypeName.TIMESTAMP));
        }
      }

      BufferedReader reader = null;
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
      return;
    }

    WindowedHashMap<Object, EventWrapper, Long> insertMap; // Map where we insert this event.
    WindowedHashMap<Object, EventWrapper, Long> joinMap; // Map we pull join candidates from.
    TypedField keyField; // The field to grab from the event wrapper.
    boolean isLeft;

    if (streamName.equals(mLeftName)) {
      insertMap = mLeftMap;
      joinMap = mRightMap;
View Full Code Here
719
720
721
722
723
724
725
726
727
728
    for (String fieldName : g.getFieldNames()) {
      resolveIdentifier(fieldsSymTab, fieldName, symRef, typeRef);
      Type fieldType = typeRef.item;
      AssignedSymbol fieldSym = symRef.item;
      typedFields.add(new TypedField(fieldSym.getAssignedName(), fieldType));
    }

    g.setFieldTypes(typedFields);
  }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
  private void runFnTest(String query, List<Pair<String, Object>> checkFields)
      throws IOException, InterruptedException {
    LOG.info("Running function test: " + query);
    MemStreamBuilder streamBuilder = new MemStreamBuilder("memstream");

    streamBuilder.addField(new TypedField("a", Type.getPrimitive(Type.TypeName.INT)));
    streamBuilder.addField(new TypedField("b", Type.getNullable(Type.TypeName.INT)));
    streamBuilder.addEvent("1,2");
    streamBuilder.addEvent("3,-4");
    StreamSymbol stream = streamBuilder.build();
    getSymbolTable().addSymbol(stream);
View Full Code Here
TOP

Related Classes of com.odiago.flumebase.parser.TypedField

  • com.odiago.flumebase.exec.FileSourceElement$EventGenThread
  • com.odiago.flumebase.exec.HashJoinElement
  • com.odiago.flumebase.exec.OutputElement
  • com.odiago.flumebase.exec.ProjectionElement
  • com.odiago.flumebase.exec.TestFunctions
  • com.odiago.flumebase.exec.TestGroupBy
  • com.odiago.flumebase.exec.TestJoin
  • com.odiago.flumebase.exec.TestSelect
  • com.odiago.flumebase.exec.TestStreamOutput
  • com.odiago.flumebase.exec.TestThroughput
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.