Examples of VoltBulkLoader

org.voltdb.client.VoltBulkLoader.VoltBulkLoader
VoltBulkLoader is meant to run for long periods of time. Multiple threads can operate on a single instance of the VoltBulkLoader to feed and bulk load a single table. It is also possible for multiple instances of the VoltBulkLoader to operate concurrently no the same table or different tables as long as they share the same Client instance. All instances of VoltBulkLoader using a common Client share a pool of threads dedicated to feeding individual partitions. There is also a thread dedicated to processing multi-partition tables. Finally, each instance of VoltBulkLoader has a thread, dedicated to processing failed bulk inserts. If multiple threads are inserting rows into the same VoltBulkLoader instance and batch inserts are relatively frequent, using multiple VoltBulkLoaders should improve performance characteristics particularly if one thread is injecting more failed rows than other threads. If one thread is injecting too many failures, relative to other threads operating on the same table, that thread should be isolated a different process. If failed rows are a common problem, throughput can also be improved by using multiple instances of VoltBulkLoader.

Examples of org.voltdb.client.VoltBulkLoader.VoltBulkLoader


    @Override
    public VoltBulkLoader getNewBulkLoader(String tableName, int maxBatchSize, BulkLoaderFailureCallBack blfcb) throws Exception
    {
        synchronized(m_vblGlobals) {
            return new VoltBulkLoader(m_vblGlobals, tableName, maxBatchSize, blfcb);
        }
    }

View Full Code Here

Examples of org.voltdb.client.VoltBulkLoader.VoltBulkLoader

            client1 = ClientFactory.createClient();
            client1.createConnection("localhost");


            prepare();
            TestFailureCallback testCallback = new TestFailureCallback();
            VoltBulkLoader bulkLoader = client1.getNewBulkLoader("BLAH", my_batchSize, testCallback);
            if (flushInterval > 0) {
                bulkLoader.setFlushInterval(0, flushInterval);
            }
            // do the test


            VoltTable modCount;
            modCount = client1.callProcedure("@AdHoc", "SELECT * FROM BLAH;").getResults()[0];
            System.out.println("data inserted to table BLAH:\n" + modCount);


            // Call validate partitioning to check if we are good.
            VoltTable valTable;
            valTable = client1.callProcedure("@ValidatePartitioning", null, null).getResults()[0];
            System.out.println("Validate for BLAH:\n" + valTable);
            while (valTable.advanceRow()) {
                long miscnt = valTable.getLong("MISPARTITIONED_ROWS");
                assertEquals(miscnt, 0);
            }


            int rowCnt=1;
            try{
                for (Object[] nextRow : my_data) {
                    Integer rowId = new Integer(rowCnt);
                    bulkLoader.insertRow(rowId, nextRow);
                    rowCnt++;
                    if (flushInterval <= 0 && (rnd.nextInt() % 30 == 0)) {
                        //  Randomly inject a flush if no timer flush is involved.
                        bulkLoader.flush();
                    }
                }
            }
            catch( Exception e) {
                System.err.print( e.getMessage() );
            }
            System.out.println(String.format("Attempted inserting %d rows", --rowCnt));


            if (flushInterval <= 0 && rnd.nextBoolean()) {
                // One in 10 tests generate a sync and VoltBulkLoader internal state verification
                bulkLoader.drain();
                assertEquals(0, bulkLoader.getOutstandingRowCount());
                assertEquals(rowCnt, bulkLoader.getCompletedRowCount());
            }
            if (flushInterval > 0) {
                //Lets get timerFlush in
                Thread.sleep(flushInterval + 500);
                bulkLoader.drain();
                //We should have everything processed callbacked.
                assertEquals(0, bulkLoader.getOutstandingRowCount());
                assertEquals(rowCnt, bulkLoader.getCompletedRowCount());
            }


            bulkLoader.close();
            assertEquals(rowCnt, bulkLoader.getCompletedRowCount());
            assertTrue(testCallback.failureRowListMatches(expectedFailList));
        }
        finally {
            if (client1 != null) client1.close();
            client1 = null;

View Full Code Here

Examples of org.voltdb.client.VoltBulkLoader.VoltBulkLoader


            prepare();
            TestFailureCallback testCallback1 = new TestFailureCallback();
            TestFailureCallback testCallback2 = new TestFailureCallback();


            VoltBulkLoader bulkLoader1 = client1.getNewBulkLoader(my_tableName1, my_batchSize1, testCallback1);
            VoltBulkLoader bulkLoader2;
            if (multipleLoaders) {
                bulkLoader2 = client2.getNewBulkLoader(my_tableName2, my_batchSize2, testCallback2);
                if (!multipleClients && sameTable) {
                    assert(bulkLoader1.getMaxBatchSize() == Math.min(my_batchSize1, my_batchSize2));
                    assert(bulkLoader1.getMaxBatchSize() == bulkLoader2.getMaxBatchSize());
                }
            }
            else
                bulkLoader2 = bulkLoader1;


            // do the test


            VoltTable modCount1;
            modCount1 = client1.callProcedure("@AdHoc", "SELECT * FROM " + my_tableName1 + ";").getResults()[0];
            System.out.println("data inserted to table " + my_tableName1 + ":\n" + modCount1);


            // Call validate partitioning to check if we are good.
            VoltTable valTable1;
            valTable1 = client1.callProcedure("@ValidatePartitioning", null, null).getResults()[0];
            System.out.println("Validate for " + my_tableName1 + ":\n" + valTable1);
            while (valTable1.advanceRow()) {
                long miscnt = valTable1.getLong("MISPARTITIONED_ROWS");
                assertEquals(miscnt, 0);
            }


            if (multipleClients) {
                VoltTable modCount2;
                modCount2 = client2.callProcedure("@AdHoc", "SELECT * FROM " + my_tableName2 + ";").getResults()[0];
                System.out.println("data inserted to table " + my_tableName2 + ":\n" + modCount2);


                // Call validate partitioning to check if we are good.
                VoltTable valTable2;
                valTable2 = client2.callProcedure("@ValidatePartitioning", null, null).getResults()[0];
                System.out.println("Validate for " + my_tableName1 + ":\n" + valTable2);
                while (valTable2.advanceRow()) {
                    long miscnt = valTable2.getLong("MISPARTITIONED_ROWS");
                    assertEquals(miscnt, 0);
                }
            }


            int rowCnt1=1;
            int rowCnt2=1;
            try{
                while (rowCnt1 <= my_data1.length || rowCnt2 <= my_data2.length) {
                    if (rowCnt1 <= my_data1.length) {
                        Integer rowId = new Integer(rowCnt1);
                        bulkLoader1.insertRow(rowId, my_data1[rowCnt1-1]);
                        rowCnt1++;
//                      if (rnd.nextInt() % 30 == 0)
//                          //  Randomly inject a flush
//                          bulkLoader1.flush();
                    }
                    if (rowCnt2 <= my_data2.length) {
                        Integer rowId = new Integer(rowCnt2);
                        bulkLoader2.insertRow(rowId, my_data2[rowCnt2-1]);
                        rowCnt2++;
//                      if (rnd.nextInt() % 30 == 0)
//                          //  Randomly inject a flush
//                          bulkLoader2.flush();
                    }
                }
            }
            catch( Exception e) {
                System.err.print( e.getMessage() );
            }
            System.out.println(String.format("Attempted inserting %d rows in Table %s and %d rows in Table %s",
                    --rowCnt1, my_tableName1, --rowCnt2, my_tableName2));


            if (!abort1 && rnd.nextInt() % 4 == 0) {
                // One in 4 tests generate a sync and VoltBulkLoader internal state verification
                bulkLoader1.drain();
                assert(bulkLoader1.getOutstandingRowCount() == 0);
                assert(bulkLoader1.getCompletedRowCount() == rowCnt1);
            }
            if (multipleLoaders && !abort2 && rnd.nextInt() % 4 == 0) {
                bulkLoader2.drain();
                assert(bulkLoader2.getOutstandingRowCount() == 0);
                assert(bulkLoader2.getCompletedRowCount() == rowCnt2);
            }


            bulkLoader1.close();
            if (multipleLoaders)
                bulkLoader2.close();
            assert(abort1 || testCallback1.failureRowListMatches(expectedFailList1));
            assert(abort2 || testCallback2.failureRowListMatches(expectedFailList2));
        }
        finally {
            if (client1 != null) {

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.