Package it.cnr.isti.hpc.log

Examples of it.cnr.isti.hpc.log.ProgressLogger

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
  @SuppressWarnings("unchecked")
  public static void main(String[] args) {

    GetTitlesCLI cli = new GetTitlesCLI(args);
    ProgressLogger pl = new ProgressLogger("dumped {} titles", 10000);
    cli.openOutput();
    RecordReader<Article> reader = new RecordReader<Article>(
        cli.getInput(), new JsonRecordParser<Article>(Article.class));

    reader.filter(ShortTitleFilter.FEWER_THAN_THREE,
        TypeFilter.MAIN_CATEGORY_TEMPLATE,
        RedirectFilter.FILTER_OUT_REDIRECTS);

    for (Article a : reader) {
      pl.up();
      cli.writeLineInOutput(a.getTitleInWikistyle());

    }
    cli.closeOutput();
  }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
  }

  public static void main(String[] args) {
    JsonToLineCLI cli = new JsonToLineCLI(args);
    String format = cli.getParam("format");
    ProgressLogger pl = new ProgressLogger("dumped {} articles ", 100000);
    RecordReader<Article> reader = new RecordReader<Article>(
        cli.getInput(), new JsonRecordParser<Article>(Article.class));
    cli.openOutput();

    for (Article a : reader) {
      pl.up();
      for (int i = 0; i < format.length(); i++) {
        char c = format.charAt(i);
        if (c == 'w') {
          cli.writeInOutput(a.getWikiTitle());
          continue;
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
  }

  public static void main(String[] args) {

    GetDumpSummaryCLI cli = new GetDumpSummaryCLI(args);
    ProgressLogger pl = new ProgressLogger("dumped {} titles", 10000);
    cli.openOutput();
    RecordReader<Article> reader = new RecordReader<Article>(
        cli.getInput(), new JsonRecordParser<Article>(Article.class));
    ArticleSummarizer summarizer = new ArticleSummarizer();

    for (Article a : reader) {
      pl.up();
      cli.writeInOutput(a.getTypeName());
      cli.writeInOutput(TAB);
      cli.writeInOutput(String.valueOf(a.getWikiId()));
      cli.writeInOutput(TAB);
      cli.writeInOutput(a.getWikiTitle());
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    cli.openOutput();
   
    RecordReader<Article> reader = new RecordReader<Article>(cli.getInput(),
        new JsonRecordParser<Article>(Article.class));

    ProgressLogger pl = new ProgressLogger("processed {} articles",100000);

    for (Article a : reader) {
      pl.up();
      if (a.getTitleInWikistyle() == null) {
        logger.warn("current title is null, ignoring ",
            a.getTitleInWikistyle());
        continue;
      }
View Full Code Here
TOP

Related Classes of it.cnr.isti.hpc.log.ProgressLogger

  • it.cnr.isti.hpc.wikipedia.cli.GetDumpSummaryCLI
  • it.cnr.isti.hpc.wikipedia.cli.GetTitleAndTypeCLI
  • it.cnr.isti.hpc.wikipedia.cli.GetTitlesCLI
  • it.cnr.isti.hpc.wikipedia.cli.JsonToLineCLI
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.