Package org.apache.crunch.io

Examples of org.apache.crunch.io.FormatBundle.configure()

100
101
102
103
104
105
106
107
108
109
110
  @Test
  public void configureBundleSpecific(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.SPECIFIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(SpecificData.class)));
  }

  @Test
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
  @Test
  public void configureBundleGeneric(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.GENERIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(GenericData.class)));
  }

  @Test
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
  @Test
  public void configureBundleReflect(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.REFLECT.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(ReflectData.class)));
  }

  @Test
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
    ReaderWriterFactory fakeFactory = new FakeReaderWriterFactory();
    AvroMode mode = AvroMode.SPECIFIC.withFactory(fakeFactory);
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    mode.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getFactory(), is(instanceOf(FakeReaderWriterFactory.class)));
  }

  @Test
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
        Integer nodeIndex = nodeEntry.getKey();
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      if (format instanceof FileInputFormat && !conf.getBoolean(RuntimeParameters.DISABLE_COMBINE_FILE, true)) {
        format = new CrunchCombineFileInputFormat<Object, Object>(job);
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
  @Test
  public void configureBundleSpecific(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.SPECIFIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(SpecificData.class)));
  }

  @Test
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
  @Test
  public void configureBundleGeneric(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.GENERIC.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(GenericData.class)));
  }

  @Test
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
  @Test
  public void configureBundleReflect(){
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    AvroMode.REFLECT.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getData(), is(instanceOf(ReflectData.class)));
  }

  @Test
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    ReaderWriterFactory fakeFactory = new FakeReaderWriterFactory();
    AvroMode mode = AvroMode.SPECIFIC.withFactory(fakeFactory);
    FormatBundle bundle = FormatBundle.forInput(AvroInputFormat.class);
    Configuration config = new Configuration();
    mode.configure(bundle);
    bundle.configure(config);
    AvroMode returnedMode = AvroMode.fromConfiguration(config);
    assertThat(returnedMode.getFactory(), is(instanceOf(FakeReaderWriterFactory.class)));
  }

  @Test
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.