Package org.apache.crunch.types

Examples of org.apache.crunch.types.PTableType

255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
  }
 
  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
      if (at.getSchema().equals(Schema.create(Schema.Type.STRING))) {
        return true;
      }
    } else if (WritableTypeFamily.getInstance().equals(ptype.getFamily())) {
      if (ptype instanceof PTableType) {
        PTableType ptt = (PTableType) ptype;
        return isText(ptt.getKeyType()) && isText(ptt.getValueType());
      } else {
        return isText(ptype);
      }
    }
    return false;
View Full Code Here
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
   * @param value the PType of the value in the table
   * @return PTableType for reading and writing avro tables
   */
  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
  public <T> PType<T> as(PType<T> ptype) {
    if (ptype instanceof AvroType || ptype instanceof AvroGroupedTableType) {
      return ptype;
    }
    if (ptype instanceof PGroupedTableType) {
      PTableType ptt = ((PGroupedTableType) ptype).getTableType();
      return new AvroGroupedTableType((AvroTableType) as(ptt));
    }
    Class<T> typeClass = ptype.getTypeClass();
    PType<T> prim = Avros.getPrimitiveType(typeClass);
    if (prim != null) {
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
    if (ptype instanceof WritableType || ptype instanceof WritableTableType
        || ptype instanceof WritableGroupedTableType) {
      return ptype;
    }
    if (ptype instanceof PGroupedTableType) {
      PTableType ptt = ((PGroupedTableType) ptype).getTableType();
      return new WritableGroupedTableType((WritableTableType) as(ptt));
    }
    PType<T> prim = Writables.getPrimitiveType(ptype.getTypeClass());
    if (prim != null) {
      return prim;
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
  public SeqFileReaderFactory(PType<T> ptype) {
    this.converter = ptype.getConverter();
    this.mapFn = ptype.getInputMapFn();
    if (ptype instanceof PTableType) {
      PTableType ptt = (PTableType) ptype;
      this.key = SeqFileHelper.newInstance(ptt.getKeyType(), null);
      this.value = SeqFileHelper.newInstance(ptt.getValueType(), null);
    } else {
      this.key = NullWritable.get();
      this.value = SeqFileHelper.newInstance(ptype, null);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.types.PTableType

  • org.apache.crunch.impl.mem.MemPipeline
  • org.apache.crunch.io.seq.SeqFileReaderFactory
  • org.apache.crunch.io.text.TextFileTarget
  • org.apache.crunch.types.avro.Avros
  • org.apache.crunch.types.avro.AvroTypeFamily
  • org.apache.crunch.types.writable.WritableTypeFamily
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.