Package cc.mallet.pipe.iterator

Examples of cc.mallet.pipe.iterator.LineGroupIterator

123
124
125
126
127
128
129
130
131
132
133
134
135
136
    Pipe pipe = new SerialPipes (new Pipe[] {
        basePipe,
        new TokenSequence2FeatureVectorSequence (true, true),
    });

    Iterator<Instance> trainSource = new LineGroupIterator (new FileReader (trainFile.value), Pattern.compile ("^\\s*$"), true);
    Iterator<Instance> testSource;
    if (testFile.wasInvoked ()) {
      testSource = new LineGroupIterator (new FileReader (testFile.value), Pattern.compile ("^\\s*$"), true);
    } else {
      testSource = null;
    }

    InstanceList training = new InstanceList (pipe);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
  public void testFromSerialization () throws IOException, ClassNotFoundException
  {
    Pipe p = new GenericAcrfData2TokenSequence ();
    InstanceList training = new InstanceList (p);
    training.addThruPipe (new LineGroupIterator (new StringReader (sampleData), Pattern.compile ("^$"), true));

    Pipe p2 = (Pipe) TestSerializable.cloneViaSerialization (p);

    InstanceList l1 = new InstanceList (p);
    l1.addThruPipe (new LineGroupIterator (new StringReader (sampleData2), Pattern.compile ("^$"), true));
    InstanceList l2 = new InstanceList (p2);
    l2.addThruPipe (new LineGroupIterator (new StringReader (sampleData2), Pattern.compile ("^$"), true));

    // the readResolve alphabet thing doesn't kick in on first deserialization
    assertTrue (p.getTargetAlphabet () != p2.getTargetAlphabet ());

    assertEquals (1, l1.size ());
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
  public void testFixedNumLabels () throws IOException, ClassNotFoundException
  {
    Pipe p = new GenericAcrfData2TokenSequence (2);
    InstanceList training = new InstanceList (p);
    training.addThruPipe (new LineGroupIterator (new StringReader (sampleFixedData), Pattern.compile ("^$"), true));

    assertEquals (1, training.size ());

    Instance inst1 = training.get (0);
    LabelsSequence ls1 = (LabelsSequence) inst1.getTarget ();
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
  {
    GenericAcrfData2TokenSequence p = new GenericAcrfData2TokenSequence (2);
    p.setLabelsAtEnd (true);

    InstanceList training = new InstanceList (p);
    training.addThruPipe (new LineGroupIterator (new StringReader (labelsAtEndData), Pattern.compile ("^$"), true));

    assertEquals (1, training.size ());

    Instance inst1 = training.get (0);
    StringTokenization toks = (StringTokenization) inst1.getData ();
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    GenericAcrfData2TokenSequence p = new GenericAcrfData2TokenSequence (2);
    p.setFeaturesIncludeToken(false);
    p.setIncludeTokenText(false);

    InstanceList training = new InstanceList (p);
    training.addThruPipe (new LineGroupIterator (new StringReader (sampleFixedData), Pattern.compile ("^$"), true));

    assertEquals (1, training.size ());

    Instance inst1 = training.get (0);
View Full Code Here
944
945
946
947
948
949
950
951
952
953
954
        new TokenSequenceParseFeatureString(false), new TokenText(),
        new TokenSequence2FeatureVectorSequence(true, false),
        new Target2LabelSequence(), new PrintInputAndTarget(), });

    InstanceList data = new InstanceList(p);
    data.addThruPipe(new LineGroupIterator(new StringReader(toy), Pattern
        .compile("\n"), true));

    CRF crf = new CRF(p, null);
    crf.print();
    crf.addStatesForLabelsConnectedAsIn(data);
View Full Code Here
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
    if (trainOption.value)
    {
      p.setTargetProcessing(true);
      trainingData = new InstanceList(p);
      trainingData.addThruPipe(
          new LineGroupIterator(trainingFile,
            Pattern.compile("^\\s*$"), true));
      logger.info
        ("Number of features in training data: "+p.getDataAlphabet().size());
      if (testOption.value != null)
      {
        if (testFile != null)
        {
          testData = new InstanceList(p);
          testData.addThruPipe(
              new LineGroupIterator(testFile,
                Pattern.compile("^\\s*$"), true));
        } else
        {
          Random r = new Random (randomSeedOption.value);
          InstanceList[] trainingLists =
            trainingData.split(
                r, new double[] {trainingFractionOption.value,
                  1-trainingFractionOption.value});
          trainingData = trainingLists[0];
          testData = trainingLists[1];
        }
      }
    } else if (testOption.value != null)
    {
      p.setTargetProcessing(true);
      testData = new InstanceList(p);
      testData.addThruPipe(
          new LineGroupIterator(testFile,
            Pattern.compile("^\\s*$"), true));
    } else
    {
      p.setTargetProcessing(false);
      testData = new InstanceList(p);
      testData.addThruPipe(
          new LineGroupIterator(testFile,
            Pattern.compile("^\\s*$"), true));
    }
    logger.info ("Number of predicates: "+p.getDataAlphabet().size());
   
   
View Full Code Here
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604

    if (trainOption.value) {
      p.setTargetProcessing(true);
      trainingData = new InstanceList(p);
      trainingData.addThruPipe(new LineGroupIterator(trainingFile,
                               Pattern.compile("^\\s*$"), true));

      logger.info("Number of features in training data: "+p.getDataAlphabet().size());

      if (testOption.value != null) {
        if (testFile != null) {
          testData = new InstanceList(p);
          testData.addThruPipe(new LineGroupIterator(testFile,
                                 Pattern.compile("^\\s*$"), true));
        }
        else {
          Random r = new Random (randomSeedOption.value);
          InstanceList[] trainingLists =
            trainingData.split(r, new double[] {trainingFractionOption.value,
                              1 - trainingFractionOption.value});
          trainingData = trainingLists[0];
          testData = trainingLists[1];
        }
      }
    }
    else if (testOption.value != null) {
      p.setTargetProcessing(true);
      testData = new InstanceList(p);
      testData.addThruPipe(new LineGroupIterator(testFile,
                             Pattern.compile("^\\s*$"), true));
    }
    else {
        p.setTargetProcessing(false);
        testData = new InstanceList(p);
        testData.addThruPipe(
                   new LineGroupIterator(testFile,
                               Pattern.compile("^\\s*$"), true));
    }
    logger.info ("Number of predicates: "+p.getDataAlphabet().size());
   
   
View Full Code Here
213
214
215
216
217
218
219
220
  private static Iterator<Instance> constructIterator (File trainFile, File dataDir, boolean isList) throws IOException
  {
    if (isList) {
      return new FileListIterator (trainFile, dataDir, null, null, true);
    } else {
      return new LineGroupIterator (new FileReader (trainFile), Pattern.compile ("^\\s*$"), true);
    }
  }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
        new GenericAcrfData2TokenSequence (2),
        new TokenSequence2FeatureVectorSequence (true, true),
    });

    InstanceList training = new InstanceList (pipe);
    training.addThruPipe (new LineGroupIterator (new FileReader (trainFile),
                                         Pattern.compile ("\\s*"),
                                         true));

    InstanceList testing = new InstanceList (pipe);
    training.addThruPipe (new LineGroupIterator (new FileReader (testFile),
                                         Pattern.compile ("\\s*"),
                                         true));

    ACRF.Template[] tmpls = new ACRF.Template[] {
            new ACRF.BigramTemplate (0),
View Full Code Here
TOP

Related Classes of cc.mallet.pipe.iterator.LineGroupIterator

  • cc.mallet.fst.semi_supervised.tui.SimpleTaggerWithConstraints
  • cc.mallet.fst.SimpleTagger
  • cc.mallet.fst.SimpleTaggerStdin
  • cc.mallet.fst.tests.TestCRF
  • cc.mallet.grmm.examples.SimpleCrfExample
  • cc.mallet.grmm.learning.extract.AcrfExtractorTui
  • cc.mallet.grmm.learning.GenericAcrfTui
  • cc.mallet.grmm.test.TestGenericAcrfData2TokenSequence
  • edu.washington.cs.knowitall.argumentidentifier.ArgSubstructureClassifier
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.