Package org.apache.mahout.utils.io

Examples of org.apache.mahout.utils.io.ChunkedWriter

56
57
58
59
60
61
62
63
64
65
66
public final class SequenceFilesFromMailArchives {

  private static final Logger log = LoggerFactory.getLogger(SequenceFilesFromMailArchives.class);

  public void createSequenceFiles(MailOptions options) throws IOException {
    ChunkedWriter writer = new ChunkedWriter(new Configuration(), options.getChunkSize(), new Path(options.getOutputDir()));
    MailProcessor processor = new MailProcessor(options, options.getPrefix(), writer);
    try {
      if (options.getInput().isDirectory()) {
        PrefixAdditionFilter filter = new PrefixAdditionFilter(processor, writer);
        options.getInput().listFiles(filter);
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
  public static final String BASE_INPUT_PATH         = "baseinputpath";

  private static final int MAX_JOB_SPLIT_LOCATIONS = 1000000;

  public void createSequenceFiles(MailOptions options) throws IOException {
    ChunkedWriter writer = new ChunkedWriter(getConf(), options.getChunkSize(), new Path(options.getOutputDir()));
    MailProcessor processor = new MailProcessor(options, options.getPrefix(), writer);
    try {
      if (options.getInput().isDirectory()) {
        PrefixAdditionFilter filter = new PrefixAdditionFilter(processor, writer);
        options.getInput().listFiles(filter);
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
  }

  @Override
  protected void process(FileStatus fst, Path current) throws IOException {
    FileSystem fs = getFs();
    ChunkedWriter writer = getWriter();
    if (fst.isDir()) {
      String dirPath = getPrefix() + Path.SEPARATOR + current.getName() + Path.SEPARATOR + fst.getPath().getName();
      fs.listStatus(fst.getPath(),
                    new PrefixAdditionFilter(getConf(), dirPath, getOptions(), writer, getCharset(), fs));
    } else {
      InputStream in = null;
      try {
        in = fs.open(fst.getPath());

        StringBuilder file = new StringBuilder();
        for (String aFit : new FileLineIterable(in, getCharset(), false)) {
          file.append(aFit).append('\n');
        }
        String name = current.getName().equals(fst.getPath().getName())
            ? current.getName()
            : current.getName() + Path.SEPARATOR + fst.getPath().getName();
        writer.write(getPrefix() + Path.SEPARATOR + name, file.toString());
      } finally {
        Closeables.close(in, false);
      }
    }
  }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
    throws IOException, InterruptedException, NoSuchMethodException {
    // Running sequentially
    Charset charset = Charset.forName(getOption(CHARSET_OPTION[0]));
    String keyPrefix = getOption(KEY_PREFIX_OPTION[0]);
    FileSystem fs = FileSystem.get(input.toUri(), conf);
    ChunkedWriter writer = new ChunkedWriter(conf, Integer.parseInt(options.get(CHUNK_SIZE_OPTION[0])), output);

    try {
      SequenceFilesFromDirectoryFilter pathFilter;
      String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]);
      if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    throws IOException, InterruptedException, NoSuchMethodException {
    // Running sequentially
    Charset charset = Charset.forName(getOption(CHARSET_OPTION[0]));
    String keyPrefix = getOption(KEY_PREFIX_OPTION[0]);
    FileSystem fs = FileSystem.get(input.toUri(), conf);
    ChunkedWriter writer = new ChunkedWriter(conf, Integer.parseInt(options.get(CHUNK_SIZE_OPTION[0])), output);

    try {
      SequenceFilesFromDirectoryFilter pathFilter;
      String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]);
      if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) {
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
  public static final String BASE_INPUT_PATH         = "baseinputpath";

  private static final int MAX_JOB_SPLIT_LOCATIONS = 1000000;

  public void createSequenceFiles(MailOptions options) throws IOException {
    ChunkedWriter writer = new ChunkedWriter(getConf(), options.getChunkSize(), new Path(options.getOutputDir()));
    MailProcessor processor = new MailProcessor(options, options.getPrefix(), writer);
    try {
      if (options.getInput().isDirectory()) {
        PrefixAdditionDirectoryWalker walker = new PrefixAdditionDirectoryWalker(processor, writer);
        walker.walk(options.getInput());
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
  private static final Logger log = LoggerFactory.getLogger(
      SequenceFilesFromMailArchives.class);

  public void createSequenceFiles(MailOptions options) throws IOException {
    ChunkedWriter writer = new ChunkedWriter(
        getConf(), options.getChunkSize(), new Path(options.getOutputDir()));
    MailProcessor processor = new MailProcessor(
        options, options.getPrefix(), writer);
    try {
      if (options.getInput().isDirectory()) {
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
  }

  @Override
  protected void process(FileStatus fst, Path current) throws IOException {
    FileSystem fs = getFs();
    ChunkedWriter writer = getWriter();
    if (fst.isDir()) {
      String dirPath = getPrefix() + Path.SEPARATOR + current.getName() + Path.SEPARATOR + fst.getPath().getName();
      fs.listStatus(fst.getPath(),
                    new PrefixAdditionFilter(getConf(), dirPath, getOptions(), writer, getCharset(), fs));
    } else {
      InputStream in = null;
      try {
        in = fs.open(fst.getPath());

        StringBuilder file = new StringBuilder();
        for (String aFit : new FileLineIterable(in, getCharset(), false)) {
          file.append(aFit).append('\n');
        }
        String name = current.getName().equals(fst.getPath().getName())
            ? current.getName()
            : current.getName() + Path.SEPARATOR + fst.getPath().getName();
        writer.write(getPrefix() + Path.SEPARATOR + name, file.toString());
      } finally {
        Closeables.closeQuietly(in);
      }
    }
  }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
    String keyPrefix = getOption(KEY_PREFIX_OPTION[0]);

    Charset charset = Charset.forName(getOption(CHARSET_OPTION[0]));
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(input.toUri(), conf);
    ChunkedWriter writer = new ChunkedWriter(conf, Integer.parseInt(options.get(CHUNK_SIZE_OPTION[0])), output);

    try {
      SequenceFilesFromDirectoryFilter pathFilter;
      String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]);
      if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) {
View Full Code Here
TOP

Related Classes of org.apache.mahout.utils.io.ChunkedWriter

  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.io.Text
  • org.apache.mahout.text.PrefixAdditionFilter
  • org.apache.mahout.text.SequenceFilesFromDirectory
  • org.apache.mahout.text.SequenceFilesFromMailArchives
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.