Package org.apache.hama.bsp.message.io

Examples of org.apache.hama.bsp.message.io.SpillingDataOutputBuffer$SpillingStream

209
210
211
212
213
214
215
216
217
218
219
      processor.init(conf);
    } catch (FileNotFoundException e) {
      LOG.error("Error initializing spilled data stream.", e);
      throw new RuntimeException(e);
    }
    spillOutputBuffer = new SpillingDataOutputBuffer(bufferCount, bufferSize,
        threshold, direct, processor);
    objectWritable = new ObjectWritable();
    objectWritable.setConf(conf);
    this.conf = conf;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
public class TestMessageIO extends TestCase {

  public void testNonSpillBuffer() throws Exception {

    SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer();
    Text text = new Text("Testing the spillage of spilling buffer");

    for (int i = 0; i < 100; ++i) {
      text.write(outputBuffer);
    }
    assertTrue(outputBuffer != null);
    assertTrue(outputBuffer.size() == 4000);
    assertFalse(outputBuffer.hasSpilled());
    outputBuffer.close();
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    Configuration conf = new HamaConfiguration();
    String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
        + new BigInteger(128, new SecureRandom()).toString(32);
    SpilledDataProcessor processor = new WriteSpilledDataProcessor(fileName);
    processor.init(conf);
    SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer(2,
        1024, 1024, true, processor);
    Text text = new Text("Testing the spillage of spilling buffer");
    for (int i = 0; i < 100; ++i) {
      text.write(outputBuffer);
    }

    assertTrue(outputBuffer != null);
    assertTrue(outputBuffer.size() == 4000);
    assertTrue(outputBuffer.hasSpilled());
    File f = new File(fileName);
    assertTrue(f.exists());
    assertTrue(f.delete());
    outputBuffer.close();

  }
View Full Code Here
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
      String fileName = System.getProperty("java.io.tmpdir")
          + File.separatorChar + "testSpillInputStream.txt";
      Configuration conf = new HamaConfiguration();
      SpilledDataProcessor processor = new WriteSpilledDataProcessor(fileName);
      processor.init(conf);
      SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer(2,
          1024, 1024, true, processor);
      Text text = new Text("Testing the spillage of spilling buffer");
      for (int i = 0; i < 100; ++i) {
        text.write(outputBuffer);
        outputBuffer.markRecordEnd();
      }

      assertTrue(outputBuffer != null);
      assertTrue(outputBuffer.size() == 4000);
      assertTrue(outputBuffer.hasSpilled());
      f = new File(fileName);
      assertTrue(f.exists());
      outputBuffer.close();
      assertTrue(f.length() == 4000);// + (4000 / 1024 + 1) * 4));

      SpilledDataInputBuffer inputBuffer = outputBuffer
          .getInputStreamToRead(fileName);

      for (int i = 0; i < 100; ++i) {
        text.readFields(inputBuffer);
        assertTrue("Testing the spillage of spilling buffer".equals(text
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
      processor.init(conf);
    } catch (FileNotFoundException e) {
      LOG.error("Error initializing spilled data stream.", e);
      throw new RuntimeException(e);
    }
    spillOutputBuffer = new SpillingDataOutputBuffer(bufferCount, bufferSize,
        threshold, direct, processor);
    objectWritable = new ObjectWritable();
    objectWritable.setConf(conf);
    this.conf = conf;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
public class TestMessageIO extends TestCase {

  public void testNonSpillBuffer() throws Exception {

    SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer();
    Text text = new Text("Testing the spillage of spilling buffer");

    for (int i = 0; i < 100; ++i) {
      text.write(outputBuffer);
    }
    assertTrue(outputBuffer != null);
    assertTrue(outputBuffer.size() == 4000);
    assertFalse(outputBuffer.hasSpilled());
    outputBuffer.close();
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    Configuration conf = new HamaConfiguration();
    String fileName = System.getProperty("java.io.tmpdir") + File.separatorChar
        + new BigInteger(128, new SecureRandom()).toString(32);
    SpilledDataProcessor processor = new WriteSpilledDataProcessor(fileName);
    processor.init(conf);
    SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer(2,
        1024, 1024, true, processor);
    Text text = new Text("Testing the spillage of spilling buffer");
    for (int i = 0; i < 100; ++i) {
      text.write(outputBuffer);
    }

    assertTrue(outputBuffer != null);
    assertTrue(outputBuffer.size() == 4000);
    assertTrue(outputBuffer.hasSpilled());
    File f = new File(fileName);
    assertTrue(f.exists());
    assertTrue(f.delete());
    outputBuffer.close();

  }
View Full Code Here
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
      String fileName = System.getProperty("java.io.tmpdir")
          + File.separatorChar + "testSpillInputStream.txt";
      Configuration conf = new HamaConfiguration();
      SpilledDataProcessor processor = new WriteSpilledDataProcessor(fileName);
      processor.init(conf);
      SpillingDataOutputBuffer outputBuffer = new SpillingDataOutputBuffer(2,
          1024, 1024, true, processor);
      Text text = new Text("Testing the spillage of spilling buffer");
      for (int i = 0; i < 100; ++i) {
        text.write(outputBuffer);
        outputBuffer.markRecordEnd();
      }

      assertTrue(outputBuffer != null);
      assertTrue(outputBuffer.size() == 4000);
      assertTrue(outputBuffer.hasSpilled());
      f = new File(fileName);
      assertTrue(f.exists());
      outputBuffer.close();
      assertTrue(f.length() == 4000);// + (4000 / 1024 + 1) * 4));

      SpilledDataInputBuffer inputBuffer = outputBuffer
          .getInputStreamToRead(fileName);

      for (int i = 0; i < 100; ++i) {
        text.readFields(inputBuffer);
        assertTrue("Testing the spillage of spilling buffer".equals(text
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
      processor.init(conf);
    } catch (FileNotFoundException e) {
      LOG.error("Error initializing spilled data stream.", e);
      throw new RuntimeException(e);
    }
    spillOutputBuffer = new SpillingDataOutputBuffer(bufferCount, bufferSize,
        threshold, direct, processor);
    objectWritable = new ObjectWritable();
    objectWritable.setConf(conf);
    this.conf = conf;
  }
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
      processor.init(conf);
    } catch (FileNotFoundException e) {
      LOG.error("Error initializing spilled data stream.", e);
      throw new RuntimeException(e);
    }
    spillOutputBuffer = new SpillingDataOutputBuffer(bufferCount, bufferSize,
        threshold, direct, processor);
    objectWritable = new ObjectWritable();
    objectWritable.setConf(conf);
    this.conf = conf;
  }
View Full Code Here
TOP

Related Classes of org.apache.hama.bsp.message.io.SpillingDataOutputBuffer$SpillingStream

  • org.apache.hama.bsp.message.queue.SpillingQueue
  • org.apache.hama.bsp.message.TestMessageIO
  • java.util.BitSet
  • java.math.BigInteger
  • java.security.SecureRandom
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.