Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

Package org.neo4j.kernel.impl.batchinsert

Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

org.neo4j.kernel.impl.batchinsert.BatchInserter
The batch inserter drops support for transactions and concurrency in favor of insertion speed. When done using the batch inserter {@link #shutdown()} must be invoked and complete successfully for the Neo4j store to be in consistent state.
Only one thread at a time may work against the batch inserter, multiple threads performing concurrent access has to be synchronized.
Transactions are not supported so if your JVM/machine crash or you fail to invoke {@link #shutdown()} before JVM exits the Neo4j store can be considered being in non consistent state and the insertion has to be re-done from scratch.

    
    @Test
    public void proveIt() throws Exception
    {
        deleteFileOrDirectory( new File( PATH ) );
        BatchInserter inserter = new BatchInserterImpl( PATH/*, stringMap(
                "neostore.nodestore.db.mapped_memory", "300M",
                "neostore.relationshipstore.db.mapped_memory", "800M",
                "neostore.propertystore.db.mapped_memory", "100M",
                "neostore.propertystore.db.strings.mapped_memory", "100M" ) */);
        
        // Create one giant chain of nodes n1->n2->n3 where each node will have
        // an int property and each rel a long string property. This will yield
        // 5b nodes/relationships, 10b property records and 5b dynamic records.
        
        // Start off by creating the first 4 billion (or so) entities with the
        // batch inserter just to speed things up a little
        long first = inserter.getReferenceNode();
        int max = (int) pow( 2, 32 )-1000;
        Map<String, Object> nodeProperties = map( "number", 123 );
        Map<String, Object> relationshipProperties =
                map( "string", "A long string, which is longer than shortstring boundaries" );
        long i = 0;
        for ( ; i < max; i++ )
        {
            long second = inserter.createNode( nodeProperties );
            inserter.createRelationship( first, second, TYPE, relationshipProperties );
            if ( i > 0 && i % 1000000 == 0 ) System.out.println( (i/1000000) + "M" );
            first = second;
        }
        inserter.shutdown();
        System.out.println( "Switch to embedded" );
        
        // Then create the rest with embedded graph db.
        GraphDatabaseService db = new EmbeddedGraphDatabase( PATH );
        Node firstNode = db.getNodeById( first );

View Full Code Here


    @Test
    public void batchInsert()
    {
        // START SNIPPET: batchInsert
        BatchInserter inserter = new BatchInserterImpl( "target/neo4jdb-batchinsert" );
        BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider( inserter );
        BatchInserterIndex actors = indexProvider.nodeIndex( "actors", MapUtil.stringMap( "type", "exact" ) );
        actors.setCacheCapacity( "name", 100000 );


        Map<String, Object> properties = MapUtil.map( "name", "Keanu Reeves" );
        long node = inserter.createNode( properties );
        actors.add( node, properties );


        // Make sure to shut down the index provider
        indexProvider.shutdown();
        inserter.shutdown();
        // END SNIPPET: batchInsert
    }

View Full Code Here

    }
    
    @Test
    public void testSimple()
    {
        BatchInserter graphDb = newBatchInserter();
        long node1 = graphDb.createNode( null );
        long node2 = graphDb.createNode( null );
        long rel1 = graphDb.createRelationship( node1, node2, RelTypes.BATCH_TEST, 
            null );
        SimpleRelationship rel = graphDb.getRelationshipById( rel1 );
        assertEquals( rel.getStartNode(), node1 );
        assertEquals( rel.getEndNode(), node2 );
        assertEquals( RelTypes.BATCH_TEST.name(), rel.getType().name() );
        graphDb.shutdown();
    }

View Full Code Here

    }
    
    @Test
    public void testMore()
    {
        BatchInserter graphDb = newBatchInserter();
        long startNode = graphDb.createNode( properties );
        long endNodes[] = new long[25];
        Set<Long> rels = new HashSet<Long>();
        for ( int i = 0; i < 25; i++ )
        {
            endNodes[i] = graphDb.createNode( properties );
            rels.add( graphDb.createRelationship( startNode, endNodes[i], 
                relTypeArray[i % 5], properties ) );
        }
        for ( SimpleRelationship rel : graphDb.getRelationships( startNode ) )
        {
            assertTrue( rels.contains( rel.getId() ) );
            assertEquals( rel.getStartNode(), startNode );
        }
        graphDb.setNodeProperties( startNode, properties );
        graphDb.shutdown();
    }

View Full Code Here

    }
    
    @Test
    public void testBadStuff()
    {
        BatchInserter graphDb = newBatchInserter();
        long startNode = graphDb.createNode( properties );
        try
        {
            graphDb.createRelationship( startNode, startNode, relTypeArray[0], 
                    properties );
            fail( "Could create relationship with same start and end node" );
        }
        catch ( IllegalArgumentException e )
        {
            // good
        }
        finally
        {
            graphDb.shutdown();
        }
    }

View Full Code Here

    }
    
    @Test
    public void testWithGraphDbService()
    {
        BatchInserter batchInserter = newBatchInserter();
        GraphDatabaseService graphDb = batchInserter.getGraphDbService();
        Node startNode = graphDb.createNode();
        setProperties( startNode );
        Node endNodes[] = new Node[25];
        Set<Relationship> rels = new HashSet<Relationship>();
        for ( int i = 0; i < 25; i++ )

View Full Code Here

    }
    
    @Test
    public void testGraphDbServiceGetRelationships()
    {
        BatchInserter batchInserter = newBatchInserter();
        GraphDatabaseService graphDb = batchInserter.getGraphDbService();
        Node startNode = graphDb.createNode();
        for ( int i = 0; i < 5; i++ )
        {
            Node endNode = graphDb.createNode();
            startNode.createRelationshipTo( endNode, relTypeArray[i] );

View Full Code Here

        Neo4jTestCase.deleteFileOrDirectory( new File( PATH ) );
    }


    @Test
    public void testStartupAndShutdown() {
        BatchInserter inserter = new BatchInserterImpl( PATH );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(
                inserter );
        BatchInserterIndex index = provider.nodeIndex( "users",
                LuceneIndexImplementation.EXACT_CONFIG );
        long id = inserter.createNode( null );
        index.add( id, map( "name", "Joe", "other", "Schmoe" ) );
        provider.shutdown();
        provider = new LuceneBatchInserterIndexProvider(
                inserter );
        index = provider.nodeIndex( "users",
                LuceneIndexImplementation.EXACT_CONFIG );
        index.add( id, map( "name", "Joe", "other", "Schmoe" ) );
        provider.shutdown();
        inserter.shutdown();
    }

View Full Code Here

    public void testSome() throws Exception
    {
        // Different paths for each tests because there's a bug (on Windows)
        // causing _0.cfs file to stay open 
        String path = new File( PATH, "1" ).getAbsolutePath();
        BatchInserter inserter = new BatchInserterImpl( path );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(
                inserter );
        BatchInserterIndex index = provider.nodeIndex( "users", EXACT_CONFIG );
        Map<Integer, Long> ids = new HashMap<Integer, Long>();
        for ( int i = 0; i < 100; i++ )
        {
            long id = inserter.createNode( null );
            index.add( id, map( "name", "Joe" + i, "other", "Schmoe" ) );
            ids.put( i, id );
        }


        for ( int i = 0; i < 100; i++ )
        {
            assertContains( index.get( "name", "Joe" + i ), ids.get( i ) );
        }
        assertContains( index.query( "name:Joe0 AND other:Schmoe" ),
                ids.get( 0 ) );


        assertContains( index.query( "name", "Joe*" ),
                ids.values().toArray( new Long[ids.size()] ) );
        provider.shutdown();
        inserter.shutdown();


        GraphDatabaseService db = new EmbeddedGraphDatabase( path );
        assertTrue( db.index().existsForNodes( "users" ) );
        Index<Node> dbIndex = db.index().forNodes( "users" );
        for ( int i = 0; i < 100; i++ )

View Full Code Here


    @Test
    public void testFulltext()
    {
        String path = new File( PATH, "2" ).getAbsolutePath();
        BatchInserter inserter = new BatchInserterImpl( path );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(
                inserter );
        String name = "users";
        BatchInserterIndex index = provider.nodeIndex( name,
                stringMap( "type", "fulltext" ) );


        long id1 = inserter.createNode( null );
        index.add( id1, map( "name", "Mattias Persson", "email",
                "something@somewhere", "something", "bad" ) );
        long id2 = inserter.createNode( null );
        index.add( id2, map( "name", "Lars PerssoN" ) );
        index.flush();
        assertContains( index.get( "name", "Mattias Persson" ), id1 );
        assertContains( index.query( "name", "mattias" ), id1 );
        assertContains( index.query( "name", "bla" ) );
        assertContains( index.query( "name", "persson" ), id1, id2 );
        assertContains( index.query( "email", "*@*" ), id1 );
        assertContains( index.get( "something", "bad" ), id1 );
        long id3 = inserter.createNode( null );
        index.add( id3, map( "name", new String[] { "What Ever", "Anything" } ) );
        index.flush();
        assertContains( index.get( "name", "What Ever" ), id3 );
        assertContains( index.get( "name", "Anything" ), id3 );


        provider.shutdown();
        inserter.shutdown();


        GraphDatabaseService db = new EmbeddedGraphDatabase( path );
        Index<Node> dbIndex = db.index().forNodes( name );
        Node node1 = db.getNodeById( id1 );
        Node node2 = db.getNodeById( id2 );

View Full Code Here

0 1

TOP

Related Classes of org.neo4j.kernel.impl.batchinsert.BatchInserter

examples.ImdbExampleTest

org.neo4j.index.impl.lucene.TestLuceneBatchInsert

org.neo4j.kernel.impl.batchinsert.TestBatchInsert

org.neo4j.kernel.impl.core.TestProveFiveBillion

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.