Package com.thinkaurelius.faunus

Examples of com.thinkaurelius.faunus.FaunusPipeline

139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
                    "it.in_degrees = it.inE().count()\n" +
                    "it.out_degrees = it.outE().count()\n" +
                    "it.degrees = it.in_degrees + it.out_degrees\n" +
                    "}";

            FaunusPipeline pipeline = new FaunusPipeline(faunusExportGraph);
            pipeline.V().sideEffect(sideEffect);
            pipeline.done();

            logger.debug("starting export of '" + graph.getId() + "'");

            FaunusJob faunusJob = new FaunusJob(metaGraphService.getMetaGraph(), jobId, pipeline);
            faunusJob.call();

            logger.debug("finished export of '" + graph.getId() + "'");

            // Filter out all the edges
            FaunusGraph faunusImportGraph = faunusExportGraph.getNextGraph();
            faunusPipelineService.configureGraph(faunusImportGraph, new Path(tmpDir, "import"), graph);

            faunusImportGraph.setGraphOutputFormat(TitanHBaseOutputFormat.class);
            faunusImportGraph.getConf().set("faunus.graph.input.vertex-query-filter", "v.query().limit(0)");

            pipeline = new FaunusPipeline(faunusImportGraph);
            pipeline.V();
            pipeline.done();

            logger.debug("starting import of '" + graph.getId() + "'");

            faunusJob = new FaunusJob(metaGraphService.getMetaGraph(), jobId, pipeline);
            faunusJob.call();
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
        FaunusGraph faunusGraph = new FaunusGraph();
        faunusGraph.setGraphInputFormat(TitanHBaseInputFormat.class);
        faunusGraph.setGraphOutputFormat(EdgeListOutputFormat.class);

        faunusPipelineService.configureGraph(faunusGraph, exportDir, graph);
        FaunusPipeline exportPipeline = new FaunusPipeline(faunusGraph);
        exportPipeline._();

        exportPipeline.done();
        FaunusJob faunusJob = new FaunusJob(metaGraphService.getMetaGraph(), jobId, exportPipeline);
        faunusJob.call();
    }
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
        FaunusGraph faunusGraph = new FaunusGraph();
        faunusGraph.setGraphInputFormat(TitanHBaseInputFormat.class);
        faunusGraph.setGraphOutputFormat(AdjacencyFileOutputFormat.class);

        faunusPipelineService.configureGraph(faunusGraph, exportDir, graph);
        FaunusPipeline exportPipeline = new FaunusPipeline(faunusGraph);
        exportPipeline._();

        exportPipeline.done();
        FaunusJob faunusJob = new FaunusJob(metaGraphService.getMetaGraph(), jobId, exportPipeline);
        faunusJob.call();
    }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
    public Object call(final Object[] args) {
        final Object result = args[0];
        final Iterator itty;
        if (result instanceof FaunusPipeline) {
            try {
                final FaunusPipeline pipeline = (FaunusPipeline) result;
                pipeline.submit();
                final FileSystem hdfs = FileSystem.get(pipeline.getGraph().getConf());
                final Path output = HDFSTools.getOutputsFinalJob(hdfs, pipeline.getGraph().getOutputLocation().toString());
                itty = new TextFileLineIterator(hdfs, hdfs.globStatus(new Path(output.toString() + "/" + Tokens.SIDEEFFECT + "*")), LINES);
            } catch (Exception e) {
                throw new RuntimeException(e.getMessage(), e);
            }
        } else {
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
        assertEquals(compiler.jobs.get(0).getCombinerClass(), null);
        assertEquals(compiler.jobs.get(0).getReducerClass(), CountMapReduce.Reduce.class);
    }

    public void testJobOrder2() throws Exception {
        FaunusPipeline pipe = new FaunusPipeline(new FaunusGraph());
        FaunusCompiler compiler = pipe.getCompiler();
        assertEquals(compiler.jobs.size(), 0);
        pipe.V().out("knows")._();
        compiler.completeSequence();

        assertEquals(compiler.jobs.size(), 2);

        assertEquals(compiler.jobs.get(0).getMapperClass(), MapSequence.Map.class);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
    public void testConfigurationPersistence() throws Exception {
        Configuration conf = new Configuration();
        conf.setInt("mapred.reduce.tasks", 2);
        conf.setBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, false);
        FaunusGraph graph = new FaunusGraph(conf);
        FaunusPipeline pipeline = new FaunusPipeline(graph);
        FaunusCompiler compiler = pipeline.getCompiler();
        TitanOutputFormat outputFormat = new TitanCassandraOutputFormat();

        assertEquals(graph.getConf().getInt("mapred.reduce.tasks", -1), 2);
        assertEquals(compiler.getConf().getInt("mapred.reduce.tasks", -1), 2);
        assertFalse(graph.getConf().getBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, true));
View Full Code Here
TOP

Related Classes of com.thinkaurelius.faunus.FaunusPipeline

  • com.thinkaurelius.faunus.mapreduce.FaunusCompiler
  • com.thinkaurelius.faunus.mapreduce.FaunusCompilerTest
  • com.thinkaurelius.faunus.tinkerpop.gremlin.ResultHookClosure
  • org.lab41.dendrite.services.analysis.EdgeDegreesService
  • org.lab41.dendrite.services.analysis.GraphLabService
  • org.lab41.dendrite.services.analysis.SnapService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.