Package edu.cmu.graphchi.datablocks

Examples of edu.cmu.graphchi.datablocks.DataBlockManager

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    public static <VertexDataType> void transform(int numVertices, String baseFilename, BytesToValueConverter<VertexDataType> conv,
                                                  VertexTransformCallBack<VertexDataType> callback) throws IOException {

        VertexData<VertexDataType> vertexData = new VertexData<VertexDataType>(numVertices, baseFilename, conv, true);

        DataBlockManager blockManager = new DataBlockManager();
        vertexData.setBlockManager(blockManager);

        int CHUNK = 1000000;
        for(int i=0; i < numVertices; i += CHUNK) {
            int en = i + CHUNK;
            if (en >= numVertices) en = numVertices - 1;
            int blockId =  vertexData.load(i, en);

            Iterator<Integer> iter = vertexData.currentIterator();

            while (iter.hasNext()) {
                int j = iter.next();
                ChiPointer ptr = vertexData.getVertexValuePtr(j, blockId);
                VertexDataType oldValue = blockManager.dereference(ptr, conv);
                VertexDataType newValue = callback.map(j, oldValue);
                blockManager.writeValue(ptr, conv, newValue);
            }
            vertexData.releaseAndCommit(i, blockId);
        }

    }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    public static <VertexDataType> void foreach(int numVertices, String baseFilename, BytesToValueConverter<VertexDataType> conv,
                                                 ForeachCallback<VertexDataType> callback) throws IOException {

        VertexData<VertexDataType> vertexData = new VertexData<VertexDataType>(numVertices, baseFilename, conv, true);

        DataBlockManager blockManager = new DataBlockManager();
        vertexData.setBlockManager(blockManager);

        int CHUNK = 1000000;
        for(int i=0; i < numVertices; i += CHUNK) {
            int en = i + CHUNK;
            if (en >= numVertices) en = numVertices - 1;
            int blockId =  vertexData.load(i, en);

            Iterator<Integer> iter = vertexData.currentIterator();

            while (iter.hasNext()) {
                int j = iter.next();
                ChiPointer ptr = vertexData.getVertexValuePtr(j, blockId);
                VertexDataType value = blockManager.dereference(ptr, conv);
                callback.callback(j, value);
            }
        }
    }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
                                                                                            String baseFilename,
                                                                                            final BytesToValueConverter<VertexDataType> conv,
                                                                                            final VertexIdTranslate idTranslate) throws IOException {
        final VertexData<VertexDataType> vertexData = new VertexData<VertexDataType>(numVertices, baseFilename, conv, true);

        final DataBlockManager blockManager = new DataBlockManager();
        vertexData.setBlockManager(blockManager);

        final int CHUNK = 1000000;

        return new Iterator<VertexIdValue<VertexDataType>>() {

            int i=0, blockId;
            Iterator<Integer> curIter;

            @Override
            public boolean hasNext() {
                if (i >= numVertices - 1) return false;
                if (curIter == null || !curIter.hasNext()) {
                    int en = i + CHUNK;
                    if (en >= numVertices) en = numVertices - 1;

                    try {
                        blockId =  vertexData.load(i, en);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                    this.curIter = vertexData.currentIterator();
                }
                return true;
            }

            @Override
            public VertexIdValue next() {
                if (hasNext()) {
                    i = curIter.next();
                    ChiPointer ptr = vertexData.getVertexValuePtr(i, blockId);
                    return new VertexIdValue<VertexDataType>(idTranslate.backward(i), blockManager.dereference(ptr, conv));
                } else throw new IllegalStateException("No more elements in the iterator!");
            }

            @Override
            public void remove() {
View Full Code Here
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
     * Initializes the vertex-data file. Similar process as sharding for edges.
     * @param sparse
     * @throws IOException
     */
    private void processVertexValues(boolean sparse) throws IOException {
        DataBlockManager dataBlockManager = new DataBlockManager();
        VertexData<VertexValueType> vertexData = new VertexData<VertexValueType>(maxVertexId + 1, baseFilename,
                vertexValueTypeBytesToValueConverter, sparse);
        vertexData.setBlockManager(dataBlockManager);
        for(int p=0; p < numShards; p++) {
            int intervalSt = p * finalIdTranslate.getVertexIntervalLength();
            int intervalEn = (p + 1) * finalIdTranslate.getVertexIntervalLength() - 1;
            if (intervalEn > maxVertexId) intervalEn = maxVertexId;

            vertexShovelStreams[p].close();

            /* Read shovel and sort */
            File shovelFile = new File(vertexShovelFileName(p));
            BufferedDataInputStream in = new BufferedDataInputStream(new FileInputStream(shovelFile));

            int sizeOf = vertexValueTypeBytesToValueConverter.sizeOf();
            long[] vertexIds = new long[(int) (shovelFile.length() / (4 + sizeOf))];
            if (vertexIds.length == 0) continue;
            byte[] vertexValues = new byte[vertexIds.length * sizeOf];
            for(int i=0; i<vertexIds.length; i++) {
                int vid = in.readInt();
                int transVid = finalIdTranslate.forward(preIdTranslate.backward(vid));
                vertexIds[i] = transVid;
                in.readFully(vertexValueTemplate);
                int valueIdx = i * sizeOf;
                System.arraycopy(vertexValueTemplate, 0, vertexValues, valueIdx, sizeOf);
            }

            /* Sort */
            sortWithValues(vertexIds, vertexValues, sizeOf);  // The source id is  higher order, so sorting the longs will produce right result

            int SUBINTERVAL = 2000000;

            int iterIdx = 0;

            /* Insert into data */
            for(int subIntervalSt=intervalSt; subIntervalSt < intervalEn; subIntervalSt += SUBINTERVAL) {
                int subIntervalEn = subIntervalSt + SUBINTERVAL - 1;
                if (subIntervalEn > intervalEn) subIntervalEn = intervalEn;
                int blockId = vertexData.load(subIntervalSt, subIntervalEn);

                Iterator<Integer> iterator = vertexData.currentIterator();
                while(iterator.hasNext()) {
                    int curId = iterator.next();

                    while(iterIdx < vertexIds.length && vertexIds[iterIdx] < curId) {
                        iterIdx++;
                    }
                    if (iterIdx >= vertexIds.length) break;

                    if (curId == (int) vertexIds[iterIdx]) {
                        ChiPointer pointer = vertexData.getVertexValuePtr(curId, blockId);
                        System.arraycopy(vertexValues, iterIdx * sizeOf, vertexValueTemplate, 0, sizeOf);
                        dataBlockManager.writeValue(pointer, vertexValueTemplate);
                    } else {
                        // No vertex data for that vertex.
                    }

                }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
     */
    public GraphChiEngine(String baseFilename, int nShards) throws FileNotFoundException, IOException {
        this.baseFilename = baseFilename;
        this.nShards = nShards;
        loadIntervals();
        blockManager = new DataBlockManager();
        degreeHandler = new DegreeData(baseFilename);

        File vertexIdTranslateFile = new File(ChiFilenames.getVertexTranslateDefFile(baseFilename, nShards));
        if (vertexIdTranslateFile.exists()) {
            vertexIdTranslate = VertexIdTranslate.fromFile(vertexIdTranslateFile);
View Full Code Here
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
*/
public class TestChiVertex {

    @Test
    public void testVertexValue() {
        DataBlockManager blockMgr = new DataBlockManager();

        int blockId = blockMgr.allocateBlock(1024* 1024);

        FloatConverter floatConv = new FloatConverter();

        ChiVertex.edgeValueConverter = floatConv;
        ChiVertex.vertexValueConverter = floatConv;
        ChiVertex.blockManager = blockMgr;
        ChiVertex<Float, Float> vertex = new ChiVertex<Float, Float>(1, new VertexDegree(0, 0));
        assertEquals(vertex.getId(), 1);

        int offset = 1024;
        ChiPointer vertexDataPtr = new ChiPointer(blockId, offset);

        blockMgr.writeValue(vertexDataPtr, floatConv, 3.0f);
        vertex.setDataPtr(vertexDataPtr);

        assertEquals(vertex.getValue(), 3.0f, 1e-15);

        vertex.setValue(999.5f);

        assertEquals(vertex.getValue(), 999.5f, 1e-15f);

        /* Check the data was committed */
        assertEquals(blockMgr.dereference(vertexDataPtr, floatConv), 999.5f, 1e-15f);
    }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
        assertEquals(blockMgr.dereference(vertexDataPtr, floatConv), 999.5f, 1e-15f);
    }

    @Test
    public void testInEdges() {
        DataBlockManager blockMgr = new DataBlockManager();

        int blockId = blockMgr.allocateBlock(1024 * 1024);

        FloatConverter floatConv = new FloatConverter();
        int nInedges = 1000;

        ChiVertex.edgeValueConverter = floatConv;
        ChiVertex.vertexValueConverter = floatConv;
        ChiVertex.blockManager = blockMgr;

        ChiVertex<Float, Float> vertex = new ChiVertex<Float, Float>(5, new VertexDegree(nInedges, 0));
        assertEquals(vertex.getId(), 5);

        for(int i=0; i < nInedges; i++) {
            blockMgr.writeValue(new ChiPointer(blockId, i * 4), floatConv, (float) Math.sin(i / 2));
            vertex.addInEdge(blockId, i * 4, i * 7 + 5);
        }
        assertEquals(vertex.numOutEdges(), 0);

        assert(vertex.numInEdges() == nInedges);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        }
    }

    @Test
    public void testOutEdges() {
        DataBlockManager blockMgr = new DataBlockManager();

        int blockId = blockMgr.allocateBlock(1024 * 1024);

        FloatConverter floatConv = new FloatConverter();
        int nOutedges = 5559;

        ChiVertex.edgeValueConverter = floatConv;
        ChiVertex.vertexValueConverter = floatConv;
        ChiVertex.blockManager = blockMgr;

        ChiVertex<Float, Float> vertex = new ChiVertex<Float, Float>(5,new VertexDegree(0, nOutedges));
        assertEquals(vertex.getId(), 5);

        assertEquals(vertex.numOutEdges(), 0);

        for(int i=0; i < nOutedges; i++) {
            blockMgr.writeValue(new ChiPointer(blockId, i * 4), floatConv, (float) Math.cos(i / 2));
            vertex.addOutEdge(blockId, i * 4, i * 7 + 5);
            assertEquals(vertex.numOutEdges(), i + 1);
        }
        assertEquals(vertex.numInEdges(), 0);

View Full Code Here
TOP

Related Classes of edu.cmu.graphchi.datablocks.DataBlockManager

  • edu.cmu.graphchi.engine.GraphChiEngine
  • edu.cmu.graphchi.preprocessing.FastSharder
  • edu.cmu.graphchi.TestChiVertex
  • edu.cmu.graphchi.vertexdata.VertexAggregator
  • edu.cmu.graphchi.vertexdata.VertexTransformer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.