Package org.apache.crunch

Examples of org.apache.crunch.MapFn

250
251
252
253
254
255
256
257
    }
    return (WritableType<T, W>) writables(clazz.asSubclass(Writable.class));
  }

  public static <W extends Writable> WritableType<W, W> writables(Class<W> clazz) {
    MapFn wIdentity = IdentityFn.getInstance();
    return new WritableType<W, W>(clazz, clazz, wIdentity, wIdentity);
  }
View Full Code Here
421
422
423
424
425
426
427
428
429
    return new WritableType(clazz, TupleWritable.class, input, output, ptypes);
  }

  public static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
337
338
339
340
341
342
343
344
          "Cannot create Writable records from non-Writable class"+ clazz.getCanonicalName());
    }
  }

  public static <W extends Writable> WritableType<W, W> writables(Class<W> clazz) {
    MapFn wIdentity = IdentityFn.getInstance();
    return new WritableType(clazz, clazz, wIdentity, wIdentity);
  }
View Full Code Here
674
675
676
677
678
679
680
681
682
    return new WritableType(Union.class, UnionWritable.class, input, output, ptypes);
  }

  public static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
681
682
683
684
685
686
687
688
689
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }

  public static <S, T> PType<T> derivedImmutable(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return WritableType.immutableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
249
250
251
252
253
254
255
256
    }
    return (WritableType<T, W>) writables(clazz.asSubclass(Writable.class));
  }

  public static <W extends Writable> WritableType<W, W> writables(Class<W> clazz) {
    MapFn wIdentity = IdentityFn.getInstance();
    return new WritableType<W, W>(clazz, clazz, wIdentity, wIdentity);
  }
View Full Code Here
420
421
422
423
424
425
426
427
428
    return new WritableType(clazz, TupleWritable.class, input, output, ptypes);
  }

  public static <S, T> PType<T> derived(Class<T> clazz, MapFn<S, T> inputFn, MapFn<T, S> outputFn, PType<S> base) {
    WritableType<S, ?> wt = (WritableType<S, ?>) base;
    MapFn input = new CompositeMapFn(wt.getInputMapFn(), inputFn);
    MapFn output = new CompositeMapFn(outputFn, wt.getOutputMapFn());
    return new WritableType(clazz, wt.getSerializationClass(), input, output, base.getSubTypes().toArray(new PType[0]));
  }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    this.conf = conf;
  }

  @Override
  public Iterator<T> read(FileSystem fs, Path path) {
    MapFn mapFn = null;
    if (String.class.equals(ptype.getTypeClass())) {
      mapFn = IdentityFn.getInstance();
    } else {
      // Check for a composite MapFn for the PType.
      // Note that this won't work for Avro-- need to solve that.
      MapFn input = ptype.getInputMapFn();
      if (input instanceof CompositeMapFn) {
        mapFn = ((CompositeMapFn) input).getSecond();
      }
    }
    mapFn.setConfigurationForTest(conf);
View Full Code Here
TOP

Related Classes of org.apache.crunch.MapFn

  • org.apache.crunch.impl.spark.collect.InputCollection
  • org.apache.crunch.impl.spark.collect.InputTable
  • org.apache.crunch.io.text.LineParser
  • org.apache.crunch.io.text.TextFileReaderFactory
  • org.apache.crunch.types.orc.Orcs
  • org.apache.crunch.types.writable.Writables
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.