Package org.springframework.data.hadoop.store.output

Examples of org.springframework.data.hadoop.store.output.TextFileWriter


  @Autowired
  private ApplicationContext context;

  @Test
  public void testWriteReadManyLines() throws IOException, InterruptedException {
    TextFileWriter writer = context.getBean("writer", TextFileWriter.class);

    TestUtils.writeData(writer, DATA09ARRAY, false, false);
    writer.flush();

    TextFileReader reader = new TextFileReader(getConfiguration(), writer.getPath(), null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
    writer.close();
    reader.close();
  }
View Full Code Here


    // just empty to survive without xml configs
  }

  @Test
  public void testWriteAppendReadTextManyLines() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    writer.setAppendable(true);
    TestUtils.writeData(writer, DATA09ARRAY, false);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
    writer.close();
  }
View Full Code Here

    writer.close();
  }

  @Test
  public void testWriteAppendReopen() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    writer.setAppendable(true);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
    writer.close();
    reader.close();

    writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    writer.setAppendable(true);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, StringUtils.concatenateStringArrays(DATA09ARRAY, DATA09ARRAY));
    writer.close();
    reader.close();
  }
View Full Code Here

    // just empty to survive without xml configs
  }

  @Test
  public void testWriteReadSplitTextManyLines() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, true);

    Splitter splitter = new StaticLengthSplitter(getConfiguration(), 110l);
View Full Code Here

    assertThat(readData1.size()+readData2.size()+readData3.size(), is(30));
  }

  @Test
  public void testWriteReadBZip2WithSplitSmallData() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, Codecs.BZIP2.getCodecInfo());
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, true);

    Splitter splitter = new StaticLengthSplitter(getConfiguration(), 30l);
View Full Code Here

  @Test
  public void testWriteReadTextOneLine() throws IOException {
    String[] dataArray = new String[] { DATA10 };

    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    TestUtils.writeData(writer, dataArray);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, dataArray);
  }
View Full Code Here

    TestUtils.readDataAndAssert(reader, dataArray);
  }

  @Test
  public void testWriteReadTextManyLines() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }
View Full Code Here

    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }

  @Test
  public void testWriteReadManyLinesWithGzip() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath,
        Codecs.GZIP.getCodecInfo());
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath,
        Codecs.GZIP.getCodecInfo());
View Full Code Here

    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }

  @Test
  public void testWriteReadManyLinesWithBzip2() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath,
        Codecs.BZIP2.getCodecInfo());
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath,
        Codecs.BZIP2.getCodecInfo());
View Full Code Here

    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
  }

  @Test
  public void testWriteReadManyLinesWithGzipWithCodecNaming() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath,
        Codecs.GZIP.getCodecInfo());
    ChainedFileNamingStrategy fileNamingStrategy = new ChainedFileNamingStrategy();
    fileNamingStrategy.register(new CodecFileNamingStrategy());
    fileNamingStrategy.register(new StaticFileNamingStrategy());
    writer.setFileNamingStrategy(fileNamingStrategy);
    TestUtils.writeData(writer, DATA09ARRAY);

    TextFileReader reader = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "data.gzip"),
        Codecs.GZIP.getCodecInfo());
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
View Full Code Here

TOP

Related Classes of org.springframework.data.hadoop.store.output.TextFileWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.