Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.MapOutputFile

399
400
401
402
403
404
405
406
407
408
409
410
  public void testInterruptOnDisk() throws Exception {
    final int FETCHER = 7;
    Path p = new Path("file:///tmp/foo");
    Path pTmp = OnDiskMapOutput.getTempPath(p, FETCHER);
    FileSystem mFs = mock(FileSystem.class, RETURNS_DEEP_STUBS);
    MapOutputFile mof = mock(MapOutputFile.class);
    when(mof.getInputFileForWrite(any(TaskID.class), anyLong())).thenReturn(p);
    OnDiskMapOutput<Text,Text> odmo = spy(new OnDiskMapOutput<Text,Text>(map1ID,
        id, mm, 100L, job, mof, FETCHER, true, mFs, p));
    when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt()))
        .thenReturn(odmo);
    doNothing().when(mm).waitForResource();
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
  }

  @SuppressWarnings("unchecked")
  @Test
  public void testDiskOutputBufferLeak() throws IOException {
    MapOutputFile mof = mock(MapOutputFile.class);
    when(mof.getInputFileForWrite(any(TaskID.class), anyLong())).thenReturn(
        new Path(base, "mapoutputfile"));
    TaskAttemptID mtid = new TaskAttemptID("0", 1, TaskType.MAP, 1, 1);
    MergeManager<Text,Text> merger = mock(MergeManager.class);
    MapOutput<Text,Text> odmo = new MapOutput<Text,Text>(
        mtid, merger, 0, new JobConf(), null, 0, true, mof);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class);
    CompressionCodec mockCompressionCodec = mock(CompressionCodec.class);
    Counter mockCounter = mock(Counter.class);
    TaskStatus mockTaskStatus = mock(TaskStatus.class);
    Progress mockProgress = mock(Progress.class);
    MapOutputFile mockMapOutputFile = mock(MapOutputFile.class);
    Task mockTask = mock(Task.class);

    try {
      String [] dirs = jobConf.getLocalDirs();
      // verify that these APIs are available through super class handler
View Full Code Here
317
318
319
320
321
322
323
324
325
326
327
328
  public void testInterruptOnDisk() throws Exception {
    final int FETCHER = 7;
    Path p = new Path("file:///tmp/foo");
    Path pTmp = OnDiskMapOutput.getTempPath(p, FETCHER);
    FileSystem mFs = mock(FileSystem.class, RETURNS_DEEP_STUBS);
    MapOutputFile mof = mock(MapOutputFile.class);
    when(mof.getInputFileForWrite(any(TaskID.class), anyLong())).thenReturn(p);
    OnDiskMapOutput<Text,Text> odmo = spy(new OnDiskMapOutput<Text,Text>(map1ID,
        id, mm, 100L, job, mof, FETCHER, true, mFs, p));
    when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt()))
        .thenReturn(odmo);
    doNothing().when(mm).waitForResource();
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
    InterruptedException {
    JobConf jobConf = new JobConf();
    final int SORT_FACTOR = 5;
    jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, SORT_FACTOR);

    MapOutputFile mapOutputFile = new MROutputFiles();
    FileSystem fs = FileSystem.getLocal(jobConf);
    MergeManager<IntWritable, IntWritable> manager =
      new MergeManager<IntWritable, IntWritable>(null, jobConf, fs, null
        , null, null, null, null, null, null, null, null, null, mapOutputFile);
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
    LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class);
    CompressionCodec mockCompressionCodec = mock(CompressionCodec.class);
    Counter mockCounter = mock(Counter.class);
    TaskStatus mockTaskStatus = mock(TaskStatus.class);
    Progress mockProgress = mock(Progress.class);
    MapOutputFile mockMapOutputFile = mock(MapOutputFile.class);
    Task mockTask = mock(Task.class);

    try {
      String [] dirs = jobConf.getLocalDirs();
      // verify that these APIs are available through super class handler
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    this.combinerClass = combinerClass;
    this.combineCollector = combineCollector;
    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new MapOutputFile();
    this.mapOutputFile.setConf(jobConf);
   
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();
   
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
    InterruptedException {
    JobConf jobConf = new JobConf();
    final int SORT_FACTOR = 5;
    jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, SORT_FACTOR);

    MapOutputFile mapOutputFile = new MROutputFiles();
    FileSystem fs = FileSystem.getLocal(jobConf);
    MergeManagerImpl<IntWritable, IntWritable> manager =
      new MergeManagerImpl<IntWritable, IntWritable>(null, jobConf, fs, null
        , null, null, null, null, null, null, null, null, null, mapOutputFile);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class);
    CompressionCodec mockCompressionCodec = mock(CompressionCodec.class);
    Counter mockCounter = mock(Counter.class);
    TaskStatus mockTaskStatus = mock(TaskStatus.class);
    Progress mockProgress = mock(Progress.class);
    MapOutputFile mockMapOutputFile = mock(MapOutputFile.class);
    Task mockTask = mock(Task.class);

    try {
      String [] dirs = jobConf.getLocalDirs();
      // verify that these APIs are available through super class handler
View Full Code Here
348
349
350
351
352
353
354
355
356
357
358
359
  public void testInterruptOnDisk() throws Exception {
    final int FETCHER = 7;
    Path p = new Path("file:///tmp/foo");
    Path pTmp = OnDiskMapOutput.getTempPath(p, FETCHER);
    FileSystem mFs = mock(FileSystem.class, RETURNS_DEEP_STUBS);
    MapOutputFile mof = mock(MapOutputFile.class);
    when(mof.getInputFileForWrite(any(TaskID.class), anyLong())).thenReturn(p);
    OnDiskMapOutput<Text,Text> odmo = spy(new OnDiskMapOutput<Text,Text>(map1ID,
        id, mm, 100L, job, mof, FETCHER, true, mFs, p));
    when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt()))
        .thenReturn(odmo);
    doNothing().when(mm).waitForResource();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.MapOutputFile

  • org.apache.hadoop.mapreduce.task.reduce.MergeManager
  • org.apache.hadoop.mapreduce.task.reduce.TestFetcher
  • org.apache.hadoop.mapreduce.task.reduce.TestMapOutput
  • org.apache.hadoop.mapreduce.task.reduce.TestMergeManager
  • org.apache.hadoop.mapreduce.TestShufflePlugin
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.