Package joshua.util.io

Examples of joshua.util.io.LineReader

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
  static Logger logger = Logger.getLogger(ConfigFileConverter.class.getSimpleName());
 
  public static List<Double> readGoogleWeightsFromJoshuaConfig(String joshuaConfig) throws IOException{
    List<Double> res = new ArrayList<Double>();
    LineReader     reader = new LineReader(joshuaConfig);
    for (String line : reader) {
      line = line.trim();
     
      //comment, empty line, or parameter lines
      if (!Regex.commentOrEmptyLine.matches(line) && line.indexOf("googleBLEUWeights") != -1) {
       
        String[] fds = Regex.equalsWithSpaces.split(line);
        if (fds.length != 2) {
          logger.severe("Wrong config line: " + line);
          System.exit(1);
        }
       
        //== add new lines for models
        //googleBLEUWeights=1.0;-1.0;1;9;2
                                 
        String[] weights = fds[1].trim().split(";");//1;0.1;
        if(weights.length!=5){
          logger.severe("Wrong number of weights in line: " + line);
          System.exit(1);
        }
        for(int i=0; i<5; i ++){//0gram 1.0
          double weight = new Double(weights[i]);
          res.add(weight);
        }             
       
      }
    }
    reader.close();
    if(res.size()!=5){
      logger.severe("Wrong number of google weights, " + res.size());
      System.exit(1);
    }
    return res;   
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    JoshuaDecoder.writeConfigFile(newWeights, mertConfigTemplate, outputFile, null);
  }
 
  static public List<Double> readGoogleWeightsFromMERTConfig(String mertConfig) throws IOException{
    List<Double> res = new ArrayList<Double>();
    LineReader     reader = new LineReader(mertConfig);
    for (String line : reader) {
      line = line.trim();
     
      //comment, empty line, or parameter lines
      if (!Regex.commentOrEmptyLine.matches(line) && line.indexOf("gramMatch") != -1) {
       
        String[] fds = Regex.spaces.split(line);
        if (fds.length != 2) {
          logger.severe("Wrong config line: " + line);
          System.exit(1);
        }
       
        res.add(new Double(fds[1]));
      }
    }
    if(res.size()!=5){
      logger.severe("Wrong number of google weights, " + res.size());
      System.exit(1);
    }
    reader.close();
    return res;   
  }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
  static public void convertMertToJoshuaFormat(String mertConfig, String joshuaConfigTemplate,  String outputFile) throws IOException{
    List<Double> weights = readGoogleWeightsFromMERTConfig(mertConfig);
   

    BufferedWriter writer = FileUtility.getWriteFileStream(outputFile);
    LineReader     reader = new LineReader(joshuaConfigTemplate);
    for (String line : reader) {
      line = line.trim();
     
     
      if (!Regex.commentOrEmptyLine.matches(line) && line.indexOf("googleBLEUWeights") != -1) {
       
        String[] fds = Regex.equalsWithSpaces.split(line);
        if (fds.length != 2) {
          logger.severe("Wrong config line: " + line);
          System.exit(1);
        }
       
        //== add new lines for models
        //googleBLEUWeights=1.0;-1.0;1;9;2
        StringBuffer newLine = new StringBuffer();
        newLine.append("googleBLEUWeights=");
       
        for(int i=0; i<5; i ++){//0gram 1.0
          newLine.append(weights.get(i));
          if(i<4)
            newLine.append(";"); 
        }             
        newLine.append("\n");
        writer.write(newLine.toString());
      } else{
        writer.write(line+"\n");
      }
    }
    writer.close();
    reader.close();
   
  }
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
  public static String[] getReferenceFileNames(String configFile)  throws IOException {
   
    String[] referenceFiles= null;
   
    LineReader reader = new LineReader(configFile);
    try {
      for (String line : reader) {
        line = line.trim();
        if (Regex.commentOrEmptyLine.matches(line))
          continue;
       
        if (line.indexOf("=") == -1) { // ignore lines with "="
          String[] fds = Regex.spaces.split(line);
         
          if ("oracle".equals(fds[0]) && fds.length >= 3) { //oracle files weight                   
            referenceFiles = new String[fds.length-2];
            for(int i=0; i< referenceFiles.length; i++)
              referenceFiles[i] =  fds[i+1].trim();     
          }
        }
      }
    } finally {
      reader.close();
    }
   
    return referenceFiles;
  }
View Full Code Here
17
18
19
20
21
22
23
24
25
26
27
 
  private static Logger logger = Logger.getLogger(NbestReader.class.getSimpleName());
 
  public NbestReader(String nbestFile){
    try{
      this.nbestFileReader = new LineReader(nbestFile);
    }catch  (IOException e) {
      logger.severe("cannot opne file " + nbestFile);
      System.exit(0);
    }
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
 
  public void processWholeSet(String inputNbestFile, String outputNbestFile, String[] refFiles) throws IOException{
   
    Reader<String>[] referenceReaders = new LineReader[refFiles.length];
    for(int k=0; k<refFiles.length; k++){
      LineReader refReader = new LineReader(refFiles[k]);
      referenceReaders[k] = refReader;
    }
   
   
    NbestReader nbestReader = new NbestReader(inputNbestFile);
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
  // format: lm_file host port weight
  private void read_lm_server_lists(String f_server_lists, int num_servers, String[] l_lm_server_hosts, int[] l_lm_server_ports, double[] l_lm_server_weights) throws IOException {
   
    int count = 0;
    LineReader reader = new LineReader(f_server_lists);
    try { for (String line : reader) {
      String fname = line.trim();
      Hashtable<String,?> res_conf = read_config_file(fname);
     
      String lm_file = (String)  res_conf.get("lm_file");
      String host    = (String)  res_conf.get("hostname");
      int    port    = (Integer) res_conf.get("port");
      double weight  = (Double)  res_conf.get("weight");
     
      l_lm_server_hosts[count]   = host;
      l_lm_server_ports[count]   = port;
      l_lm_server_weights[count] = weight;
      count++;
      logger.fine("lm server: "
        + "lm_file: " + lm_file
        + "; host: " + host
        + "; port: " + port
        + "; weight: " + weight);
    } } finally { reader.close(); }
     
    if (count != num_servers) {
      throw new IllegalArgumentException("num of lm servers does not match");
    }
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
  private static Hashtable<String,?> read_config_file(String config_file)
  throws IOException {
   
    Hashtable res = new Hashtable();
   
    LineReader configReader = new LineReader(config_file);
    try { for (String line : configReader) {
      //line = line.trim().toLowerCase();
      line = line.trim();
      if (Regex.commentOrEmptyLine.matches(line)) continue;
     
      if (-1 != line.indexOf("=")) { // parameters
        String[] fds = Regex.equalsWithSpaces.split(line);
        if (fds.length != 2) {
          throw new IllegalArgumentException(
            "Wrong config line: " + line);
        }
        if ("lm_file".equals(fds[0])) {
          String lm_file = fds[1].trim();
          res.put("lm_file", lm_file);
          if (logger.isLoggable(Level.FINE))
              logger.fine(String.format("lm file: %s", lm_file));
         
        } else if ("remote_lm_server_port".equals(fds[0])) {
          int port = Integer.parseInt(fds[1]);
          res.put("port", port);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("remote_lm_server_port: %s", port));
         
        } else if ("hostname".equals(fds[0])) {
          String host_name = fds[1].trim();
          res.put("hostname", host_name);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("host name is: %s", host_name));
         
        } else if ("interpolation_weight".equals(fds[0])) {
          double interpolation_weight = Double.parseDouble(fds[1]);
          res.put("weight", interpolation_weight);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("interpolation_weightt: %s", interpolation_weight));
         
        } else {
          logger.warning("LMGrammarRemote doesn't use config line: " + line);
          //System.exit(1);
        }
      }
    } } finally { configReader.close(); }
   
    return res;
  }
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
  // BUG: this is duplicating code in JoshuaConfiguration, needs unifying
  public static void read_config_file(String config_file)
  throws IOException {
   
    LineReader configReader = new LineReader(config_file);
    try { for (String line : configReader) {
      //line = line.trim().toLowerCase();
      line = line.trim();
      if (Regex.commentOrEmptyLine.matches(line)) continue;
     
      if (line.indexOf("=") != -1) { //parameters
        String[] fds = Regex.equalsWithSpaces.split(line);
        if (fds.length != 2) {
          throw new IllegalArgumentException("Wrong config line: " + line);
        }
        if ("lm_file".equals(fds[0])) {
          lm_file = fds[1].trim();
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("lm file: %s", lm_file));
         
        } else if ("use_srilm".equals(fds[0])) {
          use_srilm = Boolean.valueOf(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("use_srilm: %s", use_srilm));
         
        } else if ("lm_ceiling_cost".equals(fds[0])) {
          lm_ceiling_cost = Double.parseDouble(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("lm_ceiling_cost: %s", lm_ceiling_cost));
         
        } else if ("use_left_euqivalent_state".equals(fds[0])) {
          use_left_euqivalent_state = Boolean.valueOf(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("use_left_euqivalent_state: %s", use_left_euqivalent_state));
         
        } else if ("use_right_euqivalent_state".equals(fds[0])) {
          use_right_euqivalent_state = Boolean.valueOf(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("use_right_euqivalent_state: %s", use_right_euqivalent_state));
         
        } else if ("order".equals(fds[0])) {
          g_lm_order = Integer.parseInt(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("g_lm_order: %s", g_lm_order));
         
        } else if ("remote_lm_server_port".equals(fds[0])) {
          port = Integer.parseInt(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("remote_lm_server_port: %s", port));
         
        } else if ("remote_symbol_tbl".equals(fds[0])) {
          remote_symbol_tbl = fds[1];
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("remote_symbol_tbl: %s", remote_symbol_tbl));
         
        } else if ("hostname".equals(fds[0])) {
          g_host_name = fds[1].trim();
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("host name is: %s", g_host_name));
         
        } else if ("interpolation_weight".equals(fds[0])) {
          interpolation_weight = Double.parseDouble(fds[1]);
          if (logger.isLoggable(Level.FINE))
            logger.fine(String.format("interpolation_weightt: %s", interpolation_weight));
         
        } else {
          logger.warning("LMServer doesn't use config line: " + line);
          //System.exit(1);
        }
      }
    } } finally { configReader.close(); }
  }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
  static int[] count(String inputFilename) throws IOException {

    int numSentences = 0;
    int numWords = 0;

    LineReader lineReader = new LineReader(inputFilename);

    for (String line : lineReader) {
      String[] sentence = line.trim().split("\\s+");
      numWords += sentence.length;
      numSentences++;
View Full Code Here
TOP

Related Classes of joshua.util.io.LineReader

  • joshua.corpus.suffix_array.SuffixArrayFactory
  • joshua.corpus.vocab.DefaultSymbol
  • joshua.corpus.vocab.Vocabulary
  • joshua.decoder.DecoderFactory
  • joshua.decoder.DecoderThread
  • joshua.decoder.ff.lm.ArpaFile
  • joshua.decoder.ff.lm.buildin_lm.LMGrammarJAVA
  • joshua.decoder.ff.lm.distributed_lm.LMGrammarRemote
  • joshua.decoder.ff.lm.distributed_lm.LMServer
  • joshua.decoder.ff.tm.GrammarReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.