Examples of TTable

Examples of edu.umd.hooka.ttables.TTable

64
65
66
67
68
69
70
71
72
73
74
    public void map(IntWritable key, PhrasePair value,
                        OutputCollector<IntWritable,Text> output,
                        Reporter reporter) throws IOException {
      if (m1 == null) {
        Path pathTTable = new Path(ttable);
        TTable tt = loadTTable(pathTTable);
        m1 = new Model1(tt, true);
      }
      cr.reset();
      Alignment a = m1.viterbiAlign(value, cr);
      out.set(a.toString());
View Full Code Here

Examples of edu.umd.hooka.ttables.TTable

456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
      //          at.write(dos);
      //          dos.close();
      //          emittedATable = true;
      //        }
      //      }
      TTable tt = new TTable_monolithic_IFAs(fileSys, ttablePath, false);
      boolean emittedATable = false;
      FileStatus[] status = fileSys.listStatus(outputPath);
      for (int i=0; i<status.length; i++){
        sLogger.info("Reading " + status[i].getPath() + ", exists? " + fileSys.exists(status[i].getPath()));
        SequenceFile.Reader reader = new SequenceFile.Reader(xjob, SequenceFile.Reader.file(status[i].getPath()));
        while (reader.next(k, t)){
          if (t.getType() == PartialCountContainer.CONTENT_ARRAY) {
            tt.set(k.get(), (IndexedFloatArray)t.getContent());
            if (k.get() % 1000 == 0) reporter.progress();
            reporter.incrCounter(MergeCounters.EWORDS, 1);
            reporter.incrCounter(MergeCounters.STATISTICS, ((IndexedFloatArray)t.getContent()).size() + 1);
          } else {
            if (emittedATable)
              throw new RuntimeException("Should only have a single ATable!");
            ATable at = (ATable)t.getContent();
            fileSys.delete(atablePath, true);
            DataOutputStream dos = new DataOutputStream(
                new BufferedOutputStream(fileSys.create(atablePath)));
            at.write(dos);
            dos.close();
            emittedATable = true;
          }
        }
        reader.close();
      }
      fileSys.delete(ttablePath, true); // delete old ttable
      tt.write();  // write new one to same location
    }
View Full Code Here

Examples of edu.umd.hooka.ttables.TTable

514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
      // the following is a race condition
      fileSys.delete(outputPath.suffix("/_logs"), true);
      SequenceFile.Reader[] readers =
        SequenceFileOutputFormat.getReaders(xjob, outputPath);
      FileReaderZip z = new FileReaderZip(readers);
      TTable tt = new TTable_monolithic_IFAs(fileSys, ttablePath, false);
      boolean emittedATable = false;
      while (z.next(k,t)) {
        if (t.getType() == PartialCountContainer.CONTENT_ARRAY) {
          tt.set(k.get(), (IndexedFloatArray)t.getContent());
          if (k.get() % 1000 == 0) reporter.progress();
          reporter.incrCounter(MergeCounters.EWORDS, 1);
          reporter.incrCounter(MergeCounters.STATISTICS, ((IndexedFloatArray)t.getContent()).size() + 1);
        } else {
          if (emittedATable)
            throw new RuntimeException("Should only have a single ATable!");
          ATable at = (ATable)t.getContent();
          fileSys.delete(atablePath, true);
          DataOutputStream dos = new DataOutputStream(
              new BufferedOutputStream(fileSys.create(atablePath)));
          at.write(dos);
          dos.close();
          emittedATable = true;
        }
      }
      fileSys.delete(ttablePath, true); // delete old ttable
      tt.write();  // write new one to same location
      output.collect(key, value);
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.