Package org.apache.crunch.io

Examples of org.apache.crunch.io.FormatBundle


    Configuration base = job.getConfiguration();
    Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);

    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
        Integer nodeIndex = nodeEntry.getKey();
        List<Path> paths = nodeEntry.getValue();
        FileInputFormat.setInputPaths(jobCopy, paths.toArray(new Path[paths.size()]));
View Full Code Here


import java.io.IOException;

public class TrevniKeySource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static <S> FormatBundle getBundle(AvroType<S> ptype) {
    FormatBundle bundle = FormatBundle.forInput(AvroTrevniKeyInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName());
    return bundle;
  }
View Full Code Here

    return inputSplit.getLocations();
  }

  public void readFields(DataInput in) throws IOException {
    nodeIndex = in.readInt();
    bundle = new FormatBundle();
    bundle.setConf(conf);
    bundle.readFields(in);
    bundle.configure(conf); // yay bootstrap!
    Class<? extends InputSplit> inputSplitClass = (Class<? extends InputSplit>) readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
View Full Code Here

  }

  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    AvroType<?> atype = (AvroType<?>) ptype;
    FormatBundle bundle = FormatBundle.forOutput(AvroOutputFormat.class);
    String schemaParam = null;
    if (name == null) {
      schemaParam = "avro.output.schema";
    } else {
      schemaParam = "avro.output.schema." + name;
    }
    bundle.set(schemaParam, atype.getSchema().toString());
    AvroMode.fromType(atype).configure(bundle);
    configureForMapReduce(job, AvroWrapper.class, NullWritable.class, bundle,
        outputPath, name);
  }
View Full Code Here

    Configuration base = job.getConfiguration();
    Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);

    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      if (format instanceof FileInputFormat && !conf.getBoolean(RuntimeParameters.DISABLE_COMBINE_FILE, true)) {
        format = new CrunchCombineFileInputFormat<Object, Object>(job);
      }
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
View Full Code Here

  public void readFields(DataInput in) throws IOException {
    if (conf == null) {
      conf = new Configuration();
    }
    nodeIndex = in.readInt();
    bundle = new FormatBundle();
    bundle.setConf(conf);
    bundle.readFields(in);
    bundle.configure(conf); // yay bootstrap!
    Class<? extends InputSplit> inputSplitClass = readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
View Full Code Here

import java.io.IOException;

public class TrevniKeySource<T> extends FileSourceImpl<T> implements ReadableSource<T> {

  private static <S> FormatBundle getBundle(AvroType<S> ptype) {
    FormatBundle bundle = FormatBundle.forInput(AvroTrevniKeyInputFormat.class)
        .set(AvroJob.INPUT_IS_REFLECT, String.valueOf(ptype.hasReflect()))
        .set(AvroJob.INPUT_SCHEMA, ptype.getSchema().toString())
        .set(Avros.REFLECT_DATA_FACTORY_CLASS, Avros.REFLECT_DATA_FACTORY.getClass().getName());
    return bundle;
  }
View Full Code Here

    assertThat(returnedMode, is(AvroMode.GENERIC));
  }

  @Test
  public void configureBundleSpecific(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.SPECIFIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(SpecificData.class)));
  }
View Full Code Here

    assertThat(returnedMode.getData(), is(instanceOf(SpecificData.class)));
  }

  @Test
  public void configureBundleGeneric(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.GENERIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(GenericData.class)));
  }
View Full Code Here

    assertThat(returnedMode.getData(), is(instanceOf(GenericData.class)));
  }

  @Test
  public void configureBundleReflect(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.REFLECT.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(ReflectData.class)));
  }
View Full Code Here

TOP

Related Classes of org.apache.crunch.io.FormatBundle

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.