Examples of edu.umd.cloud9.io.array.ArrayListOfIntsWritable

edu.umd.cloud9.io.array.ArrayListOfIntsWritable
Writable extension of the {@code ArrayListOfInts} class. This class provides an efficient datastructure to store a list of ints for MapReduce jobs. @author Jimmy Lin


      //System.out.println(key.toString());
      while (values.hasNext()) {
        valIn = values.next();
        //System.out.println(valIn.toString());
        ArrayListOfIntsWritable adjListIn = valIn.getInlinks();
        adjListIn.trimToSize();
        adjList.addUnique(adjListIn.getArray());
      }


      valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
      valOut.setARank((float) 0.0);
      valOut.setInlinks(adjList);

View Full Code Here


    public void map(LongWritable key, Text value,
        OutputCollector<IntWritable, LongWritable> output,
        Reporter reporter) throws IOException {


      ArrayListOfIntsWritable links = new ArrayListOfIntsWritable();
      String line = ((Text) value).toString();
      StringTokenizer itr = new StringTokenizer(line);
      if (itr.hasMoreTokens()) {
        itr.nextToken();
      }

View Full Code Here

        OutputCollector<IntWritable, HITSNode> output, Reporter reporter)
        throws IOException {


      mOutput = output;


      ArrayListOfIntsWritable links = new ArrayListOfIntsWritable();
      String line = ((Text) value).toString();
      StringTokenizer itr = new StringTokenizer(line);
      if (itr.hasMoreTokens()) {
        links.add(Integer.parseInt(itr.nextToken()));
        // add to HMap here
      }
      while (itr.hasMoreTokens()) {
        int curr = Integer.parseInt(itr.nextToken());
        if (adjLists.containsKey(curr)) {
          ArrayListOfIntsWritable list = adjLists.get(curr);
          list.trimToSize();
          links.trimToSize();
          //FIXME
          //list.addAll(links.getArray());
          adjLists.put(curr, list);
        } else {

View Full Code Here

      int massMessages = 0;


      // Distribute PageRank mass to neighbors (along outgoing edges).
      if (node.getAdjacenyList().size() > 0) {
        // Each neighbor gets an equal share of PageRank mass.
        ArrayListOfIntsWritable list = node.getAdjacenyList();
        float mass = node.getPageRank() - (float) StrictMath.log(list.size());


        context.getCounter(PageRank.edges).increment(list.size());


        // Iterate over neighbors.
        for (int i = 0; i < list.size(); i++) {
          neighbor.set(list.get(i));
          intermediateMass.setNodeId(list.get(i));
          intermediateMass.setType(PageRankNode.Type.Mass);
          intermediateMass.setPageRank(mass);


          // Emit messages with PageRank mass to neighbors.
          context.write(neighbor, intermediateMass);

View Full Code Here

      int massMessagesSaved = 0;


      // Distribute PageRank mass to neighbors (along outgoing edges).
      if (node.getAdjacenyList().size() > 0) {
        // Each neighbor gets an equal share of PageRank mass.
        ArrayListOfIntsWritable list = node.getAdjacenyList();
        float mass = node.getPageRank() - (float) StrictMath.log(list.size());


        context.getCounter(PageRank.edges).increment(list.size());


        // Iterate over neighbors.
        for (int i = 0; i < list.size(); i++) {
          int neighbor = list.get(i);


          if (map.containsKey(neighbor)) {
            // Already message destined for that node; add PageRank mass contribution.
            massMessagesSaved++;
            map.put(neighbor, sumLogProbs(map.get(neighbor), mass));

View Full Code Here

      while (values.hasNext()) {
        PageRankNode n = values.next();


        if (n.getType().equals(PageRankNode.Type.Structure)) {
          // This is the structure; update accordingly.
          ArrayListOfIntsWritable list = n.getAdjacenyList();
          structureReceived++;


          node.setAdjacencyList(list);
        } else {
          // This is a message that contains PageRank mass; accumulate.

View Full Code Here

      int curr;
      //auth score for a node X is sum of all hub scores from nodes linking to X
      // so for each outgoing link X1...XN, contribute this node's hub score as part of node X1...XN's auth score
      // ( total auth score will be summed in reducer)
      typeOut = HITSNode.TYPE_AUTH_MASS;
      ArrayListOfIntsWritable adjList = value.getOutlinks();
      
      for (int i = 0; i < adjList.size(); i++) {
        curr = adjList.get(i);
        valOut.setType(typeOut);
        valOut.setARank(value.getHRank());
        output.collect(new IntWritable(curr), valOut);
      }
      
      //hub score for a node X is sum of all auth scores from nodes linked from X
      // so for each incoming link X1...XN, contribute this node's auth score as part of node X1...XN's hub score
      // ( total hub score will be summed in reducer)
      typeOut = HITSNode.TYPE_HUB_MASS;
      adjList = value.getInlinks();
      
      for (int i = 0; i < adjList.size(); i++) {
        curr = adjList.get(i);
        valOut.setType(typeOut);
        valOut.setHRank(value.getARank());
        output.collect(new IntWritable(curr), valOut);
      }
    }

View Full Code Here

        OutputCollector<IntWritable, HITSNode> output, Reporter reporter)
        throws IOException {


      mOutput = output;


      ArrayListOfIntsWritable adjList;
      valOut.setNodeId(value.getNodeId());
      valOut.setType(HITSNode.TYPE_NODE_MASS);
      valOut.setARank(value.getARank());
      valOut.setHRank(value.getHRank());
      output.collect(key, valOut);


      // check type using new types
      //emit hvals to outlinks as avals






      //emit avals to inlinks as hvals




      int curr;
      
      adjList = value.getOutlinks();
      for (int i = 0; i < adjList.size(); i++) {
        curr = adjList.get(i);
        // System.out.println("[key: " + key.toString() + "] [curr: " +
        // curr + "]");
        if (rankmapA.containsKey(curr)) {
          rankmapA.put(curr, sumLogProbs(rankmapA.get(curr),
              value.getHRank()));
        } else {
          rankmapA.put(curr, value.getHRank());
        }
      }
      
      adjList = value.getInlinks();
      for (int i = 0; i < adjList.size(); i++) {
        curr = adjList.get(i);
        if (rankmapH.containsKey(curr)) {
          rankmapH.put(curr, sumLogProbs(rankmapH.get(curr),
              value.getARank()));
        } else {
          rankmapH.put(curr, value.getARank());

View Full Code Here

    }


    public void reduce(IntWritable key, Iterator<HITSNode> values,
        OutputCollector<IntWritable, HITSNode> output, Reporter reporter)
        throws IOException {
      ArrayListOfIntsWritable adjList = new ArrayListOfIntsWritable();


      float hrank = Float.NEGATIVE_INFINITY;
      float arank = Float.NEGATIVE_INFINITY;
      long pos;

View Full Code Here

  public void testSerialize() throws IOException {
    PageRankNode node1 = new PageRankNode();
    node1.setType(Type.Complete);
    node1.setNodeId(1);
    node1.setPageRank(0.1f);
    node1.setAdjacencyList(new ArrayListOfIntsWritable(new int[] {1,2,3,4,5,6}));


    byte[] bytes = node1.serialize();
    PageRankNode node2 = PageRankNode.create(bytes);


    assertEquals(0.1f, node2.getPageRank(), 10e-6);
    assertEquals(Type.Complete, node2.getType());
    ArrayListOfIntsWritable adj = node2.getAdjacenyList();
    assertEquals(6, adj.size());
    assertEquals(1, adj.get(0));
    assertEquals(2, adj.get(1));
    assertEquals(3, adj.get(2));
    assertEquals(4, adj.get(3));
    assertEquals(5, adj.get(4));
    assertEquals(6, adj.get(5));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of edu.umd.cloud9.io.array.ArrayListOfIntsWritable

bak.pcj.IntIterator

cc.mrlda.InformedPrior

edu.umd.cloud9.example.bfs.BfsNodeTest

edu.umd.cloud9.example.pagerank.PageRankNodeTest

edu.umd.cloud9.io.array.ArrayListOfIntsWritable

edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest

edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest

edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest

edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest

edu.umd.cloud9.io.map.HMapIDW

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.