Package org.archive.access.nutch.jobs.graph

Examples of org.archive.access.nutch.jobs.graph.GraphManager

150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
   * @throws IOException
   */
  public int buildGraph(Path inputSegments[], Path outputPath) throws IOException {       
 
    final SequenceFile.Writer writer=SequenceFile.createWriter(fs, conf, new Path(outputPath,GRAPH_FILE), ArquivoWebKeyValueWritable.class, NullWritable.class, SequenceFile.CompressionType.BLOCK, new DefaultCodec());   
    final GraphManager graph = new GraphManager(); 
   
    readLinks(inputSegments, new ReadLinksProcessor() {     
      public void run(String fromUrl, String toUrl) throws IOException {               
        writer.append(new ArquivoWebKeyValueWritable(graph.getId(fromUrl),graph.getId(toUrl)),NullWritable.get());       
      }
    });

    writer.close();
    return graph.numNodes();   
  }
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
   * Write file with pagerank scores  
   */
  public void writeFileScores(Path inputSegments[], Path outputFile, final double scores[]) throws IOException {
   
    final SequenceFile.Writer writer=SequenceFile.createWriter(fs, conf, outputFile, Text.class, FloatWritable.class, SequenceFile.CompressionType.BLOCK, new DefaultCodec());
    final GraphManager graph = new GraphManager();

    readLinks(inputSegments, new ReadLinksProcessor() {     
      public void run(String fromUrl, String toUrl) throws IOException {  // read urls in the same order when it created the web graph, to write scores
       
        int id;
        if (!graph.hasId(fromUrl)) {
          id=graph.getId(fromUrl);
          writer.append(new Text(fromUrl), new FloatWritable( (float)scores[id] ));               
        }
        if (!graph.hasId(toUrl)) {
          id=graph.getId(toUrl);
          writer.append(new Text(toUrl), new FloatWritable( (float)scores[id] ));               
        }
      } 
    });
     
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
   */
  public void writeFileScores2debug(Path inputSegments[], Path outputFile, final double scores[]) throws IOException {
   
    FSDataOutputStream out = fs.create(outputFile);
    final PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out)));    
    final GraphManager graph = new GraphManager();
   
    readLinks(inputSegments, new ReadLinksProcessor() {   
      public void run(String fromUrl, String toUrl) throws IOException {  // read urls in the same order when it created the web graph, to write scores
       
        int id;
        if (!graph.hasId(fromUrl)) {
          id=graph.getId(fromUrl);
          writer.println(fromUrl+" "+scores[id]);               
        }
        if (!graph.hasId(toUrl)) {
          id=graph.getId(toUrl);
          writer.println(toUrl+" "+scores[id]);               
        }
      }
    });
     
View Full Code Here
TOP

Related Classes of org.archive.access.nutch.jobs.graph.GraphManager

  • org.archive.access.nutch.jobs.NutchwaxPagerank
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.