Package org.apache.crunch.io

Examples of org.apache.crunch.io.FormatBundle

42
43
44
45
46
47
48
49
50
51
  // configuration field for specifying the separator character.
  private static final String OLD_KV_SEP = "key.value.separator.in.input.line";
  private static final String NEW_KV_SEP = "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
 
  private static FormatBundle getBundle(String sep) {
    FormatBundle bundle = FormatBundle.forInput(KeyValueTextInputFormat.class);
    bundle.set(OLD_KV_SEP, sep);
    bundle.set(NEW_KV_SEP, sep);
    return bundle;
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
  public void readFields(DataInput in) throws IOException {
    if (conf == null) {
      conf = new Configuration();
    }
    nodeIndex = in.readInt();
    bundle = new FormatBundle();
    bundle.setConf(conf);
    bundle.readFields(in);
    bundle.configure(conf); // yay bootstrap!
    Class<? extends InputSplit> inputSplitClass = readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    AvroType<?> atype = (AvroType<?>) ptype;
    FormatBundle bundle = FormatBundle.forOutput(AvroOutputFormat.class);
    String schemaParam = null;
    if (name == null) {
      schemaParam = "avro.output.schema";
    } else {
      schemaParam = "avro.output.schema." + name;
    }
    for (Map.Entry<String, String> e : extraConf.entrySet()) {
      bundle.set(e.getKey(), e.getValue());
    }
    bundle.set(schemaParam, atype.getSchema().toString());
    AvroMode.fromType(atype).configure(bundle);
    configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle,
        outputPath, name);
  }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
import org.apache.hadoop.fs.Path;

public class AvroFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static <S> FormatBundle getBundle(AvroType<S> ptype) {
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName());
    return bundle;
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    AvroType<?> atype = (AvroType<?>) ptype;
    FormatBundle bundle = FormatBundle.forOutput(AvroOutputFormat.class);
    String schemaParam = null;
    if (name == null) {
      schemaParam = "avro.output.schema";
    } else {
      schemaParam = "avro.output.schema." + name;
    }
    bundle.set(schemaParam, atype.getSchema().toString());
    Avros.configureReflectDataFactory(job.getConfiguration());
    configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle,
        outputPath, name);
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
    Configuration base = job.getConfiguration();
    Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);

    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      if (format instanceof FileInputFormat && !conf.getBoolean(RuntimeParameters.DISABLE_COMBINE_FILE, false)) {
        format = new CrunchCombineFileInputFormat<Object, Object>(job);
      }
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    return String.format("CrunchInputSplit(%s)", inputSplit);
  }

  public void readFields(DataInput in) throws IOException {
    nodeIndex = in.readInt();
    bundle = new FormatBundle();
    bundle.setConf(conf);
    bundle.readFields(in);
    bundle.configure(conf); // yay bootstrap!
    Class<? extends InputSplit> inputSplitClass = readClass(in);
    inputSplit = ReflectionUtils.newInstance(inputSplitClass, conf);
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
import org.apache.hadoop.fs.Path;

public class AvroFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static <S> FormatBundle getBundle(AvroType<S> ptype) {
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName());
    return bundle;
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
* run.
*/
public class NLineFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static FormatBundle getBundle(int linesPerTask) {
    FormatBundle bundle = FormatBundle.forInput(NLineInputFormat.class);
    bundle.set(NLineInputFormat.LINES_PER_MAP, String.valueOf(linesPerTask));
    return bundle;
  }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
  // configuration field for specifying the separator character.
  private static final String OLD_KV_SEP = "key.value.separator.in.input.line";
  private static final String NEW_KV_SEP = "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
 
  private static FormatBundle getBundle(String sep) {
    FormatBundle bundle = FormatBundle.forInput(KeyValueTextInputFormat.class);
    bundle.set(OLD_KV_SEP, sep);
    bundle.set(NEW_KV_SEP, sep);
    return bundle;
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.io.FormatBundle

  • org.apache.commons.lang.builder.HashCodeBuilder
  • org.apache.crunch.impl.mr.run.CrunchInputFormat
  • org.apache.crunch.impl.mr.run.CrunchInputSplit
  • org.apache.crunch.io.avro.AvroFileSource
  • org.apache.crunch.io.avro.AvroFileSourceTest
  • org.apache.crunch.io.avro.AvroFileTarget
  • org.apache.crunch.io.avro.AvroPathPerKeyTarget
  • org.apache.crunch.io.avro.trevni.TrevniKeySource
  • org.apache.crunch.io.parquet.AvroParquetFileTarget
  • org.apache.crunch.io.text.NLineFileSource
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.