Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.LongWritable$Comparator

56
57
58
59
60
61
62
63
64
65
66
67
68
69
    URI uri = FILE.toURI();
    SequenceFile.Writer writer
      = new SequenceFile.Writer(FileSystem.get(uri, c), c,
                                new Path(uri.toString()),
                                LongWritable.class, Text.class);
    final LongWritable key = new LongWritable();
    final Text val = new Text();
    for (int i = 0; i < COUNT; ++i) {
      key.set(i);
      val.set(Integer.toString(i));
      writer.append(key, val);
    }
    writer.close();
  }
View Full Code Here
798
799
800
801
802
803
804
805
806
807
808
                cbsyncs = 0L;
              }
            }

            Path dst = makeRelative(root, stat.getPath());
            src_writer.append(new LongWritable(stat.isDir()? 0: stat.getLen()),
                new FilePair(stat, dst));
            dst_writer.append(new Text(dst.toString()),
                new Text(stat.getPath().toString()));
          }
        }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
      Path src = new Path(srcfilelist);
      FileSystem fs = src.getFileSystem(job);
      FileStatus srcst = fs.getFileStatus(src);

      ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
      LongWritable key = new LongWritable();
      FilePair value = new FilePair();
      final long targetsize = cbsize / numSplits;
      long pos = 0L;
      long last = 0L;
      long acc = 0L;
      long cbrem = srcst.getLen();
      for (SequenceFile.Reader sl = new SequenceFile.Reader(fs, src, job);
           sl.next(key, value); last = sl.getPosition()) {
        // if adding this split would put this split past the target size,
        // cut the last split and put this next file in the next split.
        if (acc + key.get() > targetsize && acc != 0) {
          long splitsize = last - pos;
          splits.add(new FileSplit(src, pos, splitsize, job));
          cbrem -= splitsize;
          pos = last;
          acc = 0L;
        }
        acc += key.get();
      }
      if (cbrem != 0) {
        splits.add(new FileSplit(src, pos, cbrem, job));
      }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    values.add(new Time(new Date().getTime()));
    values.add(new Time(new Date().getTime()));
    values.add(new Float(1.0f));

    GenericDBWritable val = new GenericDBWritable(columns, values);
    LongWritable key = new LongWritable(1);
    mapper.map(key, val, context);

    Text outkey = new Text();
    Text outval = new Text();
    StringBuilder builder = new StringBuilder();
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
    values.add(new Time(new Date().getTime()));
    values.add(new Time(new Date().getTime()));
    values.add(new Float(1.0f));

    GenericDBWritable val = new GenericDBWritable(columns, values);
    LongWritable key = new LongWritable(1);
    mapper.map(key, val, context);

    Text outkey = new Text();
    Text outval = new Text();
    StringBuilder builder = new StringBuilder();
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
    DBInputDelimMapper mapper = new DBInputDelimMapper();
    ArrayList<ColumnInfo> columns = new ArrayList<ColumnInfo>();
    ArrayList values = new ArrayList();
    GenericDBWritable val = new GenericDBWritable(columns, values);
    LongWritable key = new LongWritable(1);
    mapper.map(key, val, context);

    Text outkey = new Text();
    Text outval = new Text();
    verify(context).write(outkey, outval);
View Full Code Here
48
49
50
51
52
53
54
55
56
57
    assertEquals(md5HashKey1, md5HashKey2);
  }

  @Test
  public void testMD5HashForLongWritableKey() throws IOException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    MD5Hash md5HashKey1 = HashUtility.getMD5Hash(key);
    MD5Hash md5HashKey2 = HashUtility.getMD5Hash(key);
    assertEquals(md5HashKey1, md5HashKey2);
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    //tableInfo.add(columnInfo4);
    tableInfo.add(columnInfo5);
   
    mapper.setTableInfo(tableInfo);
   
    mapper.map(new LongWritable(0l), new Text("1,Sam,true,84.0"), context);
   
    ArrayList values = new ArrayList();
    values.add(1l);
    values.add("Sam");
    values.add(true);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    //tableInfo.add(columnInfo4);
    tableInfo.add(columnInfo5);
   
    mapper.setTableInfo(tableInfo);
   
    mapper.map(new LongWritable(0l), new Text("1, ,true,84.0"), context);
   
    ArrayList values = new ArrayList();
    values.add(1l);
    values.add(null);
    values.add(true);
View Full Code Here
189
190
191
192
193
194
195
    //tableInfo.add(columnInfo4);
    tableInfo.add(columnInfo5);
   
    mapper.setTableInfo(tableInfo);
   
    mapper.map(new LongWritable(0l), new Text("1,Sam,true,84.0,42"), context);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.LongWritable$Comparator

  • bbejeck.mapred.aggregation.AverageTemperatureCombiningMapperTest
  • bbejeck.mapred.aggregation.AverageTemperatureMapperTest
  • bbejeck.mapred.aggregation.AverageTemperatureMapReduceTest
  • bbejeck.mapred.coocurrance.PairsOccurrenceMapperTest
  • cbcb.kmulus.metrics.QueryCountMapper
  • co.nubetech.hiho.dedup.TestDedupJob
  • co.nubetech.hiho.mapreduce.TestDBInputAvroMapper
  • co.nubetech.hiho.mapreduce.TestDBInputDelimMapper
  • co.nubetech.hiho.mapreduce.TestGenericDBLoadDataMapper
  • co.nubetech.hiho.merge.TestMergeJob
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.