Package com.thinkaurelius.faunus

Examples of com.thinkaurelius.faunus.FaunusGraph

123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
        Path tmpDir = new Path("dendrite/tmp/" + UUID.randomUUID() + "/");
        fs.mkdirs(tmpDir);
        fs.deleteOnExit(tmpDir);

        try {
            FaunusGraph faunusExportGraph = new FaunusGraph();
            faunusPipelineService.configureGraph(faunusExportGraph, new Path(tmpDir, "export"), graph);

            faunusExportGraph.setGraphInputFormat(TitanHBaseInputFormat.class);

            // FIXME: https://github.com/thinkaurelius/faunus/issues/170. The sequence file would be more efficient,
            // but it doesn't seem to support vertex query filters.
            //faunusExportGraph.setGraphOutputFormat(SequenceFileOutputFormat.class);
            faunusExportGraph.setGraphOutputFormat(GraphSONOutputFormat.class);

            String sideEffect = "{ it ->\n" +
                    "it.in_degrees = it.inE().count()\n" +
                    "it.out_degrees = it.outE().count()\n" +
                    "it.degrees = it.in_degrees + it.out_degrees\n" +
                    "}";

            FaunusPipeline pipeline = new FaunusPipeline(faunusExportGraph);
            pipeline.V().sideEffect(sideEffect);
            pipeline.done();

            logger.debug("starting export of '" + graph.getId() + "'");

            FaunusJob faunusJob = new FaunusJob(metaGraphService.getMetaGraph(), jobId, pipeline);
            faunusJob.call();

            logger.debug("finished export of '" + graph.getId() + "'");

            // Filter out all the edges
            FaunusGraph faunusImportGraph = faunusExportGraph.getNextGraph();
            faunusPipelineService.configureGraph(faunusImportGraph, new Path(tmpDir, "import"), graph);

            faunusImportGraph.setGraphOutputFormat(TitanHBaseOutputFormat.class);
            faunusImportGraph.getConf().set("faunus.graph.input.vertex-query-filter", "v.query().limit(0)");

            pipeline = new FaunusPipeline(faunusImportGraph);
            pipeline.V();
            pipeline.done();
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
            logger.debug("finished snap analysis of '" + graph.getId() + "'");
        }
    }

    private void runExport(DendriteGraph graph, JobMetadata.Id jobId, Path exportDir) throws Exception {
        FaunusGraph faunusGraph = new FaunusGraph();
        faunusGraph.setGraphInputFormat(TitanHBaseInputFormat.class);
        faunusGraph.setGraphOutputFormat(EdgeListOutputFormat.class);

        faunusPipelineService.configureGraph(faunusGraph, exportDir, graph);
        FaunusPipeline exportPipeline = new FaunusPipeline(faunusGraph);
        exportPipeline._();
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
            logger.debug("finished graphlab analysis of '" + graph.getId() + "'");
        }
    }

    private void runExport(DendriteGraph graph, JobMetadata.Id jobId, Path exportDir) throws Exception {
        FaunusGraph faunusGraph = new FaunusGraph();
        faunusGraph.setGraphInputFormat(TitanHBaseInputFormat.class);
        faunusGraph.setGraphOutputFormat(AdjacencyFileOutputFormat.class);

        faunusPipelineService.configureGraph(faunusGraph, exportDir, graph);
        FaunusPipeline exportPipeline = new FaunusPipeline(faunusGraph);
        exportPipeline._();
View Full Code Here
80
81
82
83
84
85
86
87
88
        // override rexster.xml values with those from the job request
        applyToConfiguration(configuration, configOverrides);

        writeConfigurationToLog(configuration);

        final FaunusGraph graph = new FaunusGraph(configuration);
        engine.put("g", graph);
        return engine;
    }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
        for (Map.Entry<Object, Object> entry : commandLineConfiguration.entrySet()) {
            conf.set(entry.getKey().toString(), entry.getValue().toString());
        }

        final FaunusGremlinScriptEngine scriptEngine = new FaunusGremlinScriptEngine();
        scriptEngine.put("g", new FaunusGraph(conf));
        Object result = scriptEngine.eval(script);
        if (result.getClass().equals(FaunusPipeline.class))
            ((FaunusPipeline) result).submit(script, true);
        System.exit(0);
    }
View Full Code Here
17
18
19
20
21
22
23
24
25
26
27
28
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public class FaunusCompilerTest extends TestCase {

    public void testGlobalConfigurations() {
        FaunusGraph graph = new FaunusGraph();
        graph.getConf().setInt("a_property", 2);
        FaunusCompiler compiler = new FaunusCompiler(graph);
        assertEquals(compiler.getConf().getInt("a_property", -1), 2);
        assertEquals(compiler.getConf().getInt("b_property", -1), -1);
        compiler.addMap(IdentityMap.Map.class, NullWritable.class, FaunusVertex.class, new Configuration());
        compiler.completeSequence();
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
        assertEquals(compiler.getConf().getInt("a_property", -1), 2);
        assertEquals(compiler.getConf().getInt("b_property", -1), -1);
    }

    public void testJobListSize() {
        FaunusCompiler compiler = new FaunusCompiler(new FaunusGraph());
        assertEquals(compiler.jobs.size(), 0);
        compiler.addMap(IdentityMap.Map.class, NullWritable.class, FaunusVertex.class, new Configuration());
        assertEquals(compiler.jobs.size(), 0);
        compiler.addMapReduce(CountMapReduce.Map.class, null, CountMapReduce.Reduce.class, NullWritable.class, FaunusVertex.class, NullWritable.class, FaunusVertex.class, new Configuration());
        assertEquals(compiler.jobs.size(), 1);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
        compiler.addMapReduce(CountMapReduce.Map.class, null, CountMapReduce.Reduce.class, NullWritable.class, FaunusVertex.class, NullWritable.class, FaunusVertex.class, new Configuration());
        assertEquals(compiler.jobs.size(), 2);
    }

    public void testJobOrder() throws Exception {
        FaunusCompiler compiler = new FaunusCompiler(new FaunusGraph());
        assertEquals(compiler.jobs.size(), 0);
        compiler.addMap(IdentityMap.Map.class, NullWritable.class, FaunusVertex.class, new Configuration());
        assertEquals(compiler.jobs.size(), 0);
        compiler.addMapReduce(CountMapReduce.Map.class, null, CountMapReduce.Reduce.class, NullWritable.class, FaunusVertex.class, NullWritable.class, FaunusVertex.class, new Configuration());
        assertEquals(compiler.jobs.size(), 1);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        assertEquals(compiler.jobs.get(0).getCombinerClass(), null);
        assertEquals(compiler.jobs.get(0).getReducerClass(), CountMapReduce.Reduce.class);
    }

    public void testJobOrder2() throws Exception {
        FaunusPipeline pipe = new FaunusPipeline(new FaunusGraph());
        FaunusCompiler compiler = pipe.getCompiler();
        assertEquals(compiler.jobs.size(), 0);
        pipe.V().out("knows")._();
        compiler.completeSequence();
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    public void testConfigurationPersistence() throws Exception {
        Configuration conf = new Configuration();
        conf.setInt("mapred.reduce.tasks", 2);
        conf.setBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, false);
        FaunusGraph graph = new FaunusGraph(conf);
        FaunusPipeline pipeline = new FaunusPipeline(graph);
        FaunusCompiler compiler = pipeline.getCompiler();
        TitanOutputFormat outputFormat = new TitanCassandraOutputFormat();

        assertEquals(graph.getConf().getInt("mapred.reduce.tasks", -1), 2);
        assertEquals(compiler.getConf().getInt("mapred.reduce.tasks", -1), 2);
        assertFalse(graph.getConf().getBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, true));
        assertFalse(compiler.getConf().getBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, true));
        outputFormat.addMapReduceJobs(compiler);
        assertEquals(compiler.jobs.size(), 1);
        assertEquals(compiler.jobs.get(0).getConfiguration().getInt("mapred.reduce.tasks", -1), 2);
        assertFalse(compiler.jobs.get(0).getConfiguration().getBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, true));
        assertEquals(graph.getConf().getInt("mapred.reduce.tasks", -1), 2);
        assertEquals(compiler.getConf().getInt("mapred.reduce.tasks", -1), 2);

        compiler.addMap(IdentityMap.Map.class, NullWritable.class, FaunusVertex.class, IdentityMap.createConfiguration());
        compiler.completeSequence();
        assertEquals(compiler.jobs.size(), 2);
        assertEquals(compiler.jobs.get(0).getConfiguration().getInt("mapred.reduce.tasks", -1), 2);
        assertFalse(compiler.jobs.get(0).getConfiguration().getBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, true));
        assertEquals(compiler.jobs.get(1).getConfiguration().getInt("mapred.reduce.tasks", -1), 0);
        assertFalse(compiler.jobs.get(1).getConfiguration().getBoolean(TitanOutputFormat.FAUNUS_GRAPH_OUTPUT_TITAN_INFER_SCHEMA, true));
        assertEquals(graph.getConf().getInt("mapred.reduce.tasks", -1), 2);
        assertEquals(compiler.getConf().getInt("mapred.reduce.tasks", -1), 2);
    }
View Full Code Here
TOP

Related Classes of com.thinkaurelius.faunus.FaunusGraph

  • com.thinkaurelius.faunus.mapreduce.FaunusCompilerTest
  • com.thinkaurelius.faunus.mapreduce.util.EmptyConfiguration
  • com.thinkaurelius.faunus.tinkerpop.gremlin.InlineScriptExecutor
  • com.thinkaurelius.faunus.tinkerpop.rexster.FaunusEvaluationJob
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.Path
  • org.lab41.dendrite.services.analysis.EdgeDegreesService
  • org.lab41.dendrite.services.analysis.GraphLabService
  • org.lab41.dendrite.services.analysis.SnapService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.