Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.IntWritable$Comparator

172
173
174
175
176
177
178
                       Reporter reporter) throws IOException {
      int sum = 0;
      while (values.hasNext()) {
        sum += values.next().get();
      }
      output.collect(key, new IntWritable(sum));
    }
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
      }
      if (shardWriter.getNumDocs() > 0 && lastkey != null) {
        TaskID taskId = context.getTaskAttemptID().getTaskID();
        int partition = taskId.getId();
        System.out.println("###########>>>>"+partition);
        context.write(new IntWritable(partition),new Text(indexHdfsPath));
      }
      FileSystem lfs = FileSystem.getLocal(conf);
      if(lfs.exists(new Path(localtmpath)))
      {
        lfs.delete(new Path(localtmpath),true);
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
          }
      }
      return ;
    }
   
    lastkey = new IntWritable(key.getIndex());
   
    Iterator<DocumentMap> iterator = values.iterator();
    while (iterator.hasNext()) {
      if(doccount>maxDocCount||debugInfo>maxDocCount_l)
      {
View Full Code Here
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
                null, ae.getClass().getName(), ae.getMessage());
            responder.doRespond(authFailedCall);
            throw ae;
          }
          if (!isSecurityEnabled && authMethod != AuthMethod.SIMPLE) {
            doSaslReply(SaslStatus.SUCCESS, new IntWritable(
                SaslRpcServer.SWITCH_TO_SIMPLE_AUTH), null, null);
            authMethod = AuthMethod.SIMPLE;
            // client has already sent the initial Sasl message and we
            // should ignore it. Both client and server should fall back
            // to simple auth from now on.
View Full Code Here
53
54
55
56
57
58
59
    public Object getPrimitiveWritableObject(Object o) {
        IntOption object = (IntOption) o;
        if (object == null || object.isNull()) {
            return null;
        }
        return new IntWritable(object.get());
    }
View Full Code Here
14
15
16
17
18
19
20
21
22
23
24
25
26
  @Test
  public void testReduce() throws IOException {
    Text key = new Text();
    key.set("key");
    List<IntWritable> values = new ArrayList<IntWritable>();
    values.add(new IntWritable(1));
    values.add(new IntWritable(2));
    values.add(new IntWritable(3));

    JRubyReducer reducer = new JRubyReducer();
    JobConf conf = new JobConf();
    conf.set("mapred.ruby.script", "mapred.rb");
    reducer.configure(conf);
View Full Code Here
109
110
111
112
113
114
115
        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
        {
            int sum = 0;
            for (IntWritable val : values)
                sum += val.get();
            context.write(key, new IntWritable(sum));
        }
View Full Code Here
364
365
366
367
368
369
370
371
372
    conf.setStrings("io.serializations", conf.get("io.serializations"),
        "org.apache.hadoop.io.serializer.JavaSerialization");
    final MapDriver<Integer, IntWritable, IntWritable, Integer> driver = MapDriver
        .newMapDriver(new InverseMapper<Integer, IntWritable>())
        .withConfiguration(conf);
    driver.setInput(1, new IntWritable(2));
    driver.addOutput(new IntWritable(2), 1);
    driver.runTest();
  }
View Full Code Here
445
446
447
448
449
450
451
452
    final Configuration conf = new Configuration();
    conf.setStrings("io.serializations", conf.get("io.serializations"),
        "org.apache.hadoop.io.serializer.JavaSerialization");
    final ReduceDriver<Integer, IntWritable, Integer, IntWritable> driver = ReduceDriver
        .newReduceDriver(new IntSumReducer<Integer>()).withConfiguration(conf);
    driver.withInputKey(1).withInputValue(new IntWritable(2))
        .withOutput(1, new IntWritable(2)).runTest();
  }
View Full Code Here
37
38
39
40
41
42
43
44
45
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver
      = MapReduceDriver.newMapReduceDriver(new StatefulMapper(), new Reducer());
   
    mapReduceDriver.addInput(new LongWritable(1L), new Text("hello"));
    mapReduceDriver.addInput(new LongWritable(2L), new Text("schmo"));
    mapReduceDriver.withOutput(new Text("SomeKey"), new IntWritable(2));
    mapReduceDriver.runTest();

  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.IntWritable$Comparator

  • bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest
  • bbejeck.mapred.aggregation.TemperatureAveragingPair
  • bbejeck.mapred.coocurrance.PairsRelativeOccurrenceReducerTest
  • co.nubetech.hiho.dedup.TestDedupJob
  • co.nubetech.hiho.merge.TestMergeJob
  • co.nubetech.hiho.similarity.ngram.TestScoreMapper
  • co.nubetech.hiho.similarity.ngram.TestScoreReducer
  • com.alimama.mdrill.index.IndexReducer
  • com.datasalt.pangool.utils.HadoopUtils
  • com.digitalpebble.behemoth.BehemothDocument
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.