Package nom.tam.util

Examples of nom.tam.util.BufferedDataInputStream

136
137
138
139
140
141
142
143
144
145
146
147
148
149
        if (adjFile == null) {

            File compressedFile = new File(adjDataFilename + ".gz");
            if (compressedFile.exists()) {
                logger.info("Note: using compressed: " + compressedFile.getName());
                adjFile = new BufferedDataInputStream(new GZIPInputStream(new FileInputStream(compressedFile)), 1024 * 1024);

            } else {
                adjFile = new BufferedDataInputStream(new FileInputStream(adjDataFilename), 1024 * 1024);
            }
            adjFile.skipBytes(adjOffset);
        }


View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
                }
                fos.close();
            }
        } else {
            if (!vertexfile.exists()) {
                BufferedDataInputStream dis = new BufferedDataInputStream(new FileInputStream(sparseDegreeFile));
                DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(vertexfile)));

                byte[] empty = new byte[converter.sizeOf()];
                try {
                    while(true) {
                        int vertexId = Integer.reverseBytes(dis.readInt());
                        dis.skipBytes(8);
                        dos.writeInt(Integer.reverseBytes(vertexId));
                        dos.write(empty);
                    }
                } catch (EOFException err) {}
                dos.close();
                dis.close();;
            }
        }

        vertexDataFile = new RandomAccessFile(vertexfile.getAbsolutePath(), "rwd");
        vertexEn = vertexSt = 0;
View Full Code Here
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
            vertexShovelStreams[p].close();

            /* Read shovel and sort */
            File shovelFile = new File(vertexShovelFileName(p));
            BufferedDataInputStream in = new BufferedDataInputStream(new FileInputStream(shovelFile));

            int sizeOf = vertexValueTypeBytesToValueConverter.sizeOf();
            long[] vertexIds = new long[(int) (shovelFile.length() / (4 + sizeOf))];
            if (vertexIds.length == 0) continue;
            byte[] vertexValues = new byte[vertexIds.length * sizeOf];
            for(int i=0; i<vertexIds.length; i++) {
                int vid = in.readInt();
                int transVid = finalIdTranslate.forward(preIdTranslate.backward(vid));
                vertexIds[i] = transVid;
                in.readFully(vertexValueTemplate);
                int valueIdx = i * sizeOf;
                System.arraycopy(vertexValueTemplate, 0, vertexValues, valueIdx, sizeOf);
            }

            /* Sort */
 
View Full Code Here
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
        logger.info("Processing shovel " + shardNum);

        /**
         * Read the edges into memory.
         */
        BufferedDataInputStream in = new BufferedDataInputStream(new FileInputStream(shovelFile));
        for(int i=0; i<shoveled.length; i++) {
            long l = in.readLong();
            int from = getFirst(l);
            int to = getSecond(l);
            in.readFully(valueTemplate);

            int newFrom = finalIdTranslate.forward(preIdTranslate.backward(from));
            int newTo = finalIdTranslate.forward(preIdTranslate.backward(to));
            shoveled[i] = packEdges(newFrom, newTo);

            /* Edge value */
            int valueIdx = i * sizeOf;
            System.arraycopy(valueTemplate, 0, edgeValues, valueIdx, sizeOf);
            if (!memoryEfficientDegreeCount) {
                inDegrees[newTo]++;
                outDegrees[newFrom]++;
            }
        }
        numEdges += shoveled.length;

        in.close();

        /* Delete the shovel-file */
        shovelFile.delete();

        logger.info("Processing shovel " + shardNum + " ... sorting");
View Full Code Here
TOP

Related Classes of nom.tam.util.BufferedDataInputStream

  • edu.cmu.graphchi.engine.auxdata.VertexData
  • edu.cmu.graphchi.preprocessing.FastSharder
  • edu.cmu.graphchi.shards.SlidingShard
  • java.io.IOException
  • java.io.DataInputStream
  • java.io.EOFException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.