Package org.apache.giraph.utils

Examples of org.apache.giraph.utils.ExtendedDataOutput

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
  /** Number of edges. */
  private int edgeCount;

  @Override
  public void initialize(Iterable<Edge<I, E>> edges) {
    ExtendedDataOutput extendedOutputStream =
        getConf().createExtendedDataOutput();
    for (Edge<I, E> edge : edges) {
      try {
        WritableUtils.writeEdge(extendedOutputStream, edge);
      } catch (IOException e) {
        throw new IllegalStateException("initialize: Failed to serialize " +
            edge);
      }
      ++edgeCount;
    }
    serializedEdges = extendedOutputStream.getByteArray();
    serializedEdgesBytesUsed = extendedOutputStream.getPos();
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
    // since add() and iterator() work fine with a null buffer.
  }

  @Override
  public void add(Edge<I, E> edge) {
    ExtendedDataOutput extendedDataOutput =
        getConf().createExtendedDataOutput(
            serializedEdges, serializedEdgesBytesUsed);
    try {
      WritableUtils.writeEdge(extendedDataOutput, edge);
    } catch (IOException e) {
      throw new IllegalStateException("add: Failed to write to the new " +
          "byte array");
    }
    serializedEdges = extendedDataOutput.getByteArray();
    serializedEdgesBytesUsed = extendedDataOutput.getPos();
    ++edgeCount;
  }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
                            Iterable<M> messages) throws IOException {
    boolean ownsVertexId = false;
    destinationVertices.add(vertexId);
    rwLock.readLock().lock();
    try {
      ExtendedDataOutput extendedDataOutput = inMemoryMessages.get(vertexId);
      if (extendedDataOutput == null) {
        ExtendedDataOutput newExtendedDataOutput =
            config.createExtendedDataOutput();
        extendedDataOutput =
            inMemoryMessages.putIfAbsent(vertexId, newExtendedDataOutput);
        if (extendedDataOutput == null) {
          ownsVertexId = true;
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    }
  }

  @Override
  public Iterable<M> getVertexMessages(I vertexId) throws IOException {
    ExtendedDataOutput extendedDataOutput = inMemoryMessages.get(vertexId);
    if (extendedDataOutput == null) {
      extendedDataOutput = config.createExtendedDataOutput();
    }
    Iterable<M> combinedIterable = new MessageIterable(
        extendedDataOutput.getByteArray(), 0, extendedDataOutput.getPos());

    for (BasicMessageStore<I, M> fileStore : fileStores) {
      combinedIterable = Iterables.concat(combinedIterable,
          fileStore.getVertexMessages(vertexId));
    }
View Full Code Here
341
342
343
344
345
346
347
348
349
350
351
    for (int m = 0; m < mapSize; m++) {
      I vertexId = config.createVertexId();
      vertexId.readFields(in);
      int messageBytes = in.readInt();
      byte[] buf = new byte[messageBytes];
      ExtendedDataOutput extendedDataOutput =
          config.createExtendedDataOutput(buf, messageBytes);
      inMemoryMessages.put(vertexId, extendedDataOutput);
    }

    // read file stores
View Full Code Here
247
248
249
250
251
252
253
254
255
256
257
258
      throw new IllegalAccessError("addMessages: Not supported");
    }

    @Override
    public Iterable<M> getVertexMessages(I vertexId) throws IOException {
      ExtendedDataOutput extendedDataOutput = temporaryMessages.get(vertexId);
      if (extendedDataOutput == null) {
        extendedDataOutput = config.createExtendedDataOutput();
      }
      return new MessageIterable(extendedDataOutput.getByteArray(), 0,
          extendedDataOutput.getPos());
    }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
   * @return Extended data output for this vertex id (created if necessary)
   */
  private ExtendedDataOutput getExtendedDataOutput(
      ConcurrentMap<I, ExtendedDataOutput> partitionMap,
      VertexIdIterator<I> iterator) {
    ExtendedDataOutput extendedDataOutput =
        partitionMap.get(iterator.getCurrentVertexId());
    if (extendedDataOutput == null) {
      ExtendedDataOutput newExtendedDataOutput =
          config.createExtendedDataOutput();
      extendedDataOutput =
          partitionMap.putIfAbsent(
              iterator.releaseCurrentVertexId(),
              newExtendedDataOutput);
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    // expensive.  If this type of iterator is not available, fall back to
    // deserializing/serializing the messages
    if (vertexIdMessageBytesIterator != null) {
      while (vertexIdMessageBytesIterator.hasNext()) {
        vertexIdMessageBytesIterator.next();
        ExtendedDataOutput extendedDataOutput =
            getExtendedDataOutput(partitionMap, vertexIdMessageBytesIterator);

        synchronized (extendedDataOutput) {
          vertexIdMessageBytesIterator.writeCurrentMessageBytes(
              extendedDataOutput);
        }
      }
    } else {
      ByteArrayVertexIdMessages<I, M>.VertexIdMessageIterator
          vertexIdMessageIterator = messages.getVertexIdMessageIterator();
      while (vertexIdMessageIterator.hasNext()) {
        vertexIdMessageIterator.next();
        ExtendedDataOutput extendedDataOutput =
            getExtendedDataOutput(partitionMap, vertexIdMessageIterator);

        synchronized (extendedDataOutput) {
          vertexIdMessageIterator.getCurrentMessage().write(
              extendedDataOutput);
View Full Code Here
204
205
206
207
208
209
210
211
212
  protected ExtendedDataOutput readFieldsForMessages(DataInput in) throws
      IOException {
    int byteArraySize = in.readInt();
    byte[] messages = new byte[byteArraySize];
    in.readFully(messages);
    ExtendedDataOutput extendedDataOutput =
        config.createExtendedDataOutput(messages, 0);
    return extendedDataOutput;
  }
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
248
249
250
           partitionEntry : byteArrayMessagesPerVertexStore.map.entrySet()) {
        for (Map.Entry<I, ExtendedDataOutput> vertexEntry :
            partitionEntry.getValue().entrySet()) {
          ConcurrentMap<I, ExtendedDataOutput> partitionMap =
              getOrCreatePartitionMap(partitionEntry.getKey());
          ExtendedDataOutput extendedDataOutput =
              partitionMap.get(vertexEntry.getKey());
          if (extendedDataOutput == null) {
            ExtendedDataOutput newExtendedDataOutput =
                config.createExtendedDataOutput();
            extendedDataOutput =
                partitionMap.putIfAbsent(vertexEntry.getKey(),
                    newExtendedDataOutput);
            if (extendedDataOutput == null) {
View Full Code Here
TOP

Related Classes of org.apache.giraph.utils.ExtendedDataOutput

  • org.apache.giraph.comm.messages.ByteArrayMessagesPerVertexStore
  • org.apache.giraph.comm.messages.DiskBackedMessageStore
  • org.apache.giraph.comm.messages.DiskBackedMessageStore$TemporaryMessageStore
  • org.apache.giraph.comm.messages.PointerListMessagesIterable
  • org.apache.giraph.comm.messages.PointerListPerVertexStore
  • org.apache.giraph.comm.messages.primitives.long_id.LongPointerListMessageStore
  • org.apache.giraph.comm.requests.SendWorkerVerticesRequest
  • org.apache.giraph.comm.RequestTest
  • org.apache.giraph.comm.SendPartitionCache
  • org.apache.giraph.edge.ByteArrayEdges
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.