Examples of org.lilyproject.repository.api.RecordScanner

org.lilyproject.repository.api.RecordScanner
A RecordScanner allows to sequentially run over all or part of the records stored in the Lily repository.
A scanner can be obtained from {@link Repository#getScanner(RecordScan)}
.
RecordScanner's implement Iterable, so you can use a for-each loop directly on them.

When done with a scanner, be sure to call {@link #close()} on them to releasethe server-side resources.

    private Cache<String, RecordScanner> recordScannerMap;


    @GET
    @Produces("application/json")
    public EntityList<Record> get(@PathParam("id") String scanId, @DefaultValue("1") @QueryParam("batch") Long batch, @Context UriInfo uriInfo) {
        RecordScanner scanner = recordScannerMap.getIfPresent(scanId);
        if (scanner != null) {
            List<Record> records = new ArrayList<Record>();


            try {
                Record record;
                while(records.size() < batch && (record = scanner.next()) != null) {
                    records.add(record);
                }
            } catch (RepositoryException e) {
                throw new ResourceException(e, INTERNAL_SERVER_ERROR.getStatusCode());
            } catch (InterruptedException e) {

View Full Code Here

        }
    }


    @DELETE
    public Response delete(@PathParam("id") String scanId) {
        RecordScanner scanner = this.recordScannerMap.getIfPresent(scanId);
        if (scanner != null) {
            scanner.close();
            this.recordScannerMap.invalidate(scanId);
            return Response.ok().build();
        } else {
            throw new ResourceException("No scan with ID " + scanId + " found", NOT_FOUND.getStatusCode());
        }

View Full Code Here

        // Change the start/stop record IDs on the scan to the current split
        TableSplit split = (TableSplit)inputSplit;
        scan.setRawStartRecordId(split.getStartRow());
        scan.setRawStopRecordId(split.getEndRow());


        RecordScanner scanner = null;
        try {
            String hbaseTableName = Bytes.toString(split.getTableName());
            String repositoryTableName = RepoAndTableUtil.extractLilyTableName(repositoryName, hbaseTableName);
            scanner = lilyClient.getRepository(repositoryName).getTable(repositoryTableName).getScanner(scan);
        } catch (RepositoryException e) {

View Full Code Here

        cache = RecordScannerMapBuilder.createRecordScannerMap(50, TimeUnit.MILLISECONDS);
    }


    @Test
    public void testRecordScannerMapExpiration() throws Exception{
        RecordScanner scanner = new DummyRecordScanner();
        String id = "TTT";
        cache.put(id, scanner);
        Thread.sleep(100);
        RecordScanner returnScanner = cache.getIfPresent(id);
        Assert.assertNull(returnScanner);


    }

View Full Code Here


    }


    @Test
    public void testRecordScannerMapAccessExpiration() throws Exception{
        RecordScanner scanner = new DummyRecordScanner();
        String id = "QQQ";
        cache.put(id, scanner);
        Thread.sleep(10);
        Assert.assertNotNull(cache.getIfPresent(id));
        Thread.sleep(100);

View Full Code Here

        // Do a scan
        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("A"));
        scan.setStopRecordId(idGenerator.newRecordId("B"));


        RecordScanner scanner = repository.getScanner(scan);
        int i = 0;
        while (scanner.next() != null) {
            i++;
        }


        assertEquals("Number of scanned records", 10, i);
    }

View Full Code Here


        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("ZA"));
        scan.setStopRecordId(idGenerator.newRecordId("ZZ")); // stop row is exclusive


        RecordScanner scanner = repository.getScanner(scan);


        int i = 0;
        Record record;
        while ((record = scanner.next()) != null) {
            assertEquals(record.getField(fieldType1.getName()), fieldValues.get(i));
            i++;
        }
        scanner.close();
        assertEquals("Found 25 records", 25, i);


        // Same using for-each loop
        scanner = repository.getScanner(scan);
        i = 0;
        for (Record result : scanner) {
            assertEquals(result.getField(fieldType1.getName()), fieldValues.get(i));
            i++;
        }
        scanner.close();
        assertEquals("Found 25 records", 25, i);


        // Scan all records, this should give at least 26 results
        scan = new RecordScan();
        scanner = repository.getScanner(scan);
        i = 0;
        while (scanner.next() != null) {
            i++;
        }


        assertTrue("Found at least 26 records", i >= 26);
    }

View Full Code Here

        repository.create(record);


        RecordScan scan = new RecordScan();


        // normal Record scanner!
        RecordScanner scanner = repository.getScanner(scan);
        final Record next = scanner.next();
        assertNotNull(next);


        // this cast will fail
        final IdRecord casted = (IdRecord) next;
    }

View Full Code Here

                .create();


        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("PrefixScanTest"));


        RecordScanner scanner = repository.getScanner(scan);
        assertEquals(idGenerator.newRecordId("PrefixScanTest"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix1"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix2"), scanner.next().getId());
        // the scanner would run till the end of the table
        assertNotNull(scanner.next());
        scanner.close();


        scan.setRecordFilter(new RecordIdPrefixFilter(idGenerator.newRecordId("PrefixScanTest")));
        scanner = repository.getScanner(scan);
        assertEquals(idGenerator.newRecordId("PrefixScanTest"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix1"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix2"), scanner.next().getId());
        // due to the prefix filter, the scanner stops once there are no records left with the same prefix
        assertNull(scanner.next());
        scanner.close();


        //
        // When using UUID record ID's, prefix scans make less sense, except for retrieving
        // variants
        //
        RecordId uuid = idGenerator.newRecordId();
        RecordId varid1 = idGenerator.newRecordId(uuid, ImmutableMap.of("lang", "en", "year", "1999"));
        RecordId varid2 = idGenerator.newRecordId(uuid, ImmutableMap.of("lang", "fr"));


        repository.recordBuilder()
                .id(uuid)
                .recordType(recordType1.getName())
                .field(fieldType1.getName(), "foo")
                .create();


        repository.recordBuilder()
                .id(varid1)
                .recordType(recordType1.getName())
                .field(fieldType1.getName(), "foo")
                .create();


        repository.recordBuilder()
                .id(varid2)
                .recordType(recordType1.getName())
                .field(fieldType1.getName(), "foo")
                .create();


        scan = new RecordScan();
        scan.setStartRecordId(uuid);
        scan.setRecordFilter(new RecordIdPrefixFilter(uuid));
        scanner = repository.getScanner(scan);
        assertEquals(uuid, scanner.next().getId());
        assertEquals(varid1, scanner.next().getId());
        assertEquals(varid2, scanner.next().getId());
        assertNull(scanner.next());
        scanner.close();
    }

View Full Code Here


        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("ScanDeleteTest-"));
        scan.setRecordFilter(new RecordIdPrefixFilter(idGenerator.newRecordId("ScanDeleteTest-")));


        RecordScanner scanner = repository.getScanner(scan);
        assertEquals(5, countResults(scanner));
        scanner.close();


        // This is to make sure the filtering of deleted records also works when we don't
        // specify a filter on our scan.
        RecordScan singleScan = new RecordScan();
        singleScan.setStartRecordId(idGenerator.newRecordId("ScanDeleteTest-0"));
        singleScan.setStopRecordId(idGenerator.newRecordId("ScanDeleteTest-0"));


        scanner = repository.getScanner(singleScan);
        assertEquals(1, countResults(scanner));
        scanner.close();


        // Delete the records, verify the new scanner results
        repository.delete(idGenerator.newRecordId("ScanDeleteTest-0"));


        scanner = repository.getScanner(scan);
        assertEquals(4, countResults(scanner));
        scanner.close();


        scanner = repository.getScanner(singleScan);
        assertEquals(0, countResults(scanner));
        scanner.close();
    }

View Full Code Here

0 1

TOP

Related Classes of org.lilyproject.repository.api.RecordScanner

org.lilyproject.mapreduce.LilyScanInputFormat

org.lilyproject.process.test.LilyClientTest

org.lilyproject.process.test.RecordScannerMapTest

org.lilyproject.repository.impl.test.AbstractRepositoryTest

org.lilyproject.rest.RecordScanResource

org.lilyproject.tools.scanner.cli.RecordScanTool

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.