Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.LongWritable

81
82
83
84
85
86
87
88
89
90
91
  }

  @Test
  public void testReducerForLongWritableKey() throws IOException,
      InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    Text value1 = new Text("value1");
    ArrayList<Text> values = new ArrayList<Text>();
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
  }

  @Test
  public void testReducerForLongWritableKey() throws IOException,
      InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    Text value1 = new Text("value1");
    ArrayList<Text> values = new ArrayList<Text>();
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
  }
 
  @Test
  public void testDedupByLongWritableKeyWithSequenceFileInputFormat() throws Exception {
    HashMap<LongWritable, Text> inputData1 = new HashMap<LongWritable, Text>();
    inputData1.put(new LongWritable(1), new Text("Xavier Wilson,Mason Holloway,Carlos Johnston,Martin Noel,Drake Mckinney"));
    inputData1.put(new LongWritable(2), new Text("Kennedy Bailey,Jerome Perry,David Cabrera,Edan Fleming,Orlando Tyson"));
    inputData1.put(new LongWritable(3), new Text("Drake Mckinney,Murphy Baird,Theodore Lindsey,Nehru Wilcox,Harper Klein"));
    createSequenceFileInHdfs(inputData1, "/input1", "testFile1.seq");
   
    HashMap<LongWritable, Text> inputData2 = new HashMap<LongWritable, Text>();
    inputData2.put(new LongWritable(1), new Text("Zephania Bauer,Jermaine Gordon,Vincent Moon,Steven Pierce,Jasper Campos"));
    inputData2.put(new LongWritable(2), new Text("Kennedy Bailey,Plato Atkinson,Stuart Guy,Rooney Levy,Judah Benson"));
    inputData2.put(new LongWritable(4), new Text("Drake Mckinney,Murphy Baird,Theodore Lindsey,Nehru Wilcox,Harper Klein"));
    createSequenceFileInHdfs(inputData2, "/input2", "testFile2.seq");
   
    String[] args = new String[] {
        "-inputFormat", "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat",
        "-inputPath", "/input1,/input2",
        "-outputPath", "output",
        "-inputKeyClassName", "org.apache.hadoop.io.LongWritable",
        "-inputValueClassName", "org.apache.hadoop.io.Text",
        "-dedupBy", "key" };
    DedupJob job = runDedupJob(args); 
    assertEquals(6, job.getTotalRecordsRead());
    assertEquals(0, job.getBadRecords());
    assertEquals(4, job.getOutput());
    assertEquals(2, job.getDuplicateRecords());
   
   
    FileSystem outputFS = getFileSystem();
    Path outputPath = new Path(outputFS.getHomeDirectory(), "output/part-r-00000");
    Configuration conf = new Configuration();
    SequenceFile.Reader reader = new SequenceFile.Reader(outputFS, outputPath, conf);
    Writable writableKey = (Writable)
    ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    Writable writableValue = (Writable)
    ReflectionUtils.newInstance(reader.getValueClass(), conf);
    List<LongWritable> expectedOutput = new ArrayList<LongWritable>();
    expectedOutput.add(new LongWritable(1));
    expectedOutput.add(new LongWritable(2));
    expectedOutput.add(new LongWritable(3));
    expectedOutput.add(new LongWritable(4));
    int count = 0;
    while (reader.next(writableKey, writableValue)) {
      logger.debug("key and value is: " + writableKey + ", " + writableValue);
      assertTrue("Matched output " + writableKey , expectedOutput.contains(writableKey));
      count++;
View Full Code Here
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
 
 
  @Test
  public void testDedupByCustomObjectKeyWithSequenceFileInputFormat() throws Exception {   
    Student student1 = setStudent(new Text("Sam"),new Text("US"),new IntWritable(1),
        new LongWritable(9999999998l),new DoubleWritable(99.12));       
    Student student2 = setStudent(new Text("John"),new Text("AUS"),new IntWritable(2),
        new LongWritable(9999999999l),new DoubleWritable(90.12));       
    Student student3 = setStudent(new Text("Mary"),new Text("UK"),new IntWritable(3),
        new LongWritable(9999999988l),new DoubleWritable(69.12));   
    Student student4 = setStudent(new Text("Kelvin"),new Text("UK"),new IntWritable(4),
        new LongWritable(9999998888l),new DoubleWritable(59.12));
 
    HashMap<Student, Text> inputData1 = new HashMap<Student, Text>();
    inputData1.put(student1, new Text("Macon Kent,6269 Aenean St.,1-247-399-1051,08253"));
    inputData1.put(student2, new Text("Dale Zamora,521-7792 Mauris Rd.,1-214-625-6970,90510"));
    inputData1.put(student3, new Text("Charles Wood,525-9709 In Rd.,1-370-528-4758,62714"));
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    id.readFields(in);
    name = new Text();
    name.readFields(in);
    address = new Text();
    address.readFields(in);
    mobileNumber = new LongWritable();
    mobileNumber.readFields(in);
    percentage = new DoubleWritable();
    percentage.readFields(in);

  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  }

  @Test
  public void testReducerForLongWritableKey() throws IOException,
      InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
  }

  @Test
  public void testReducerForLongWritableKey() throws IOException,
      InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
View Full Code Here
34
35
36
37
38
39
40
41
42
        path, LongWritable.class, VectorWritable.class);
    long recNum = 0;
    VectorWritable vec = new VectorWritable();
    for (Vector point : points) {
      vec.set(point);
      writer.append(new LongWritable(recNum++), vec);
    }
    writer.close();
  }
View Full Code Here
766
767
768
769
770
771
772
773
774
775
776
    int expected = bodies.size();
    for(String outputFile : getAllFiles(dir)) {
      String name = (new File(outputFile)).getName();
      if(name.startsWith(prefix)) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFile), conf);
        LongWritable key = new LongWritable();
        BytesWritable value = new BytesWritable();
        while(reader.next(key, value)) {
          String body = new String(value.getBytes(), 0, value.getLength());
          if (bodies.contains(body)) {
            LOG.debug("Found event body: {}", body);
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
        if (jsonObj.get("http_result").getAsInt() == 200) {
          reporter.incrCounter(this._counterGroup, "HTTP Success", 1);

          // only output counts for pages that were successfully retrieved
          output.collect(new Text(domain), new LongWritable(1));
        }
        else {
          reporter.incrCounter(this._counterGroup, "HTTP Not Success", 1);
        }
      }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.LongWritable

  • bbejeck.mapred.aggregation.AverageTemperatureCombiningMapperTest
  • bbejeck.mapred.aggregation.AverageTemperatureMapperTest
  • bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest
  • bbejeck.mapred.coocurrance.PairsOccurrenceMapperTest
  • cbcb.kmulus.blast.BlastMapper
  • cbcb.kmulus.db.cluster.ClusterPresenceVectors$Map
  • cbcb.kmulus.metrics.QueryCountMapper
  • co.nubetech.hiho.dedup.TestDedupJob
  • co.nubetech.hiho.mapreduce.TestDBInputAvroMapper
  • co.nubetech.hiho.mapreduce.TestDBInputDelimMapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.