Package org.lilyproject.repository.api

Examples of org.lilyproject.repository.api.RecordScan

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    @Override
    public RecordScan fromJson(JsonNode nodeNode, Namespaces namespaces, LRepository repository)
            throws JsonFormatException, RepositoryException, InterruptedException {

        if (nodeNode == null || nodeNode.isNull()) {
            return new RecordScan();
        }

        if (!nodeNode.isObject()) {
            throw new JsonFormatException("Expected a json object for record scan, got: " +
                    nodeNode.getClass().getName());
        }

        ObjectNode node = (ObjectNode) nodeNode;
        RecordScan scan = new RecordScan();

        namespaces = NamespacesConverter.fromContextJson(node, namespaces);

        String startRecordId = JsonUtil.getString(node, "startRecordId", null);
        if (startRecordId != null) {
            scan.setStartRecordId(repository.getIdGenerator().fromString(startRecordId));
        }

        String stopRecordId = JsonUtil.getString(node, "stopRecordId", null);
        if (stopRecordId != null) {
            scan.setStopRecordId(repository.getIdGenerator().fromString(stopRecordId));
        }

        byte[] rawStartRecordId = JsonUtil.getBinary(node, "rawStartRecordId", null);
        if (rawStartRecordId != null) {
            scan.setRawStartRecordId(rawStartRecordId);
        }

        byte[] rawStopRecordId = JsonUtil.getBinary(node, "rawStopRecordId", null);
        if (rawStopRecordId != null) {
            scan.setRawStopRecordId(rawStopRecordId);
        }

        ObjectNode filter = JsonUtil.getObject(node, "recordFilter", null);
        if (filter != null) {
            scan.setRecordFilter(RecordFilterJsonConverters.INSTANCE.fromJson(filter, namespaces, repository,
                    RecordFilterJsonConverters.INSTANCE));
        }

        ObjectNode returnFieldsNode = JsonUtil.getObject(node, "returnFields", null);
        if (returnFieldsNode != null) {
            ReturnFields returnFields = new ReturnFields();
            returnFields.setType(ReturnFields.Type.valueOf(JsonUtil.getString(returnFieldsNode, "type")));

            ArrayNode fieldsArray = JsonUtil.getArray(returnFieldsNode, "fields", null);
            if (fieldsArray != null) {
                List<QName> fields = new ArrayList<QName>();
                for (JsonNode subFilterNode : fieldsArray) {
                    if (!subFilterNode.isTextual()) {
                        throw new JsonFormatException("ReturnFields.fields should be a string array, found: "
                                + subFilterNode.getClass().getName());
                    }
                    fields.add(QNameConverter.fromJson(subFilterNode.getTextValue(), namespaces));
                }
                returnFields.setFields(fields);
            }

            scan.setReturnFields(returnFields);
        }

        scan.setCaching(JsonUtil.getInt(node, "caching", scan.getCaching()));

        scan.setCacheBlocks(JsonUtil.getBoolean(node, "cacheBlocks", scan.getCacheBlocks()));

        return scan;
    }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
            varProps.put(vprop, null);
        }

        final ArrayList<Record> result = new ArrayList<Record>();

        final RecordScan scan = new RecordScan();
        scan.setStartRecordId(newDep.id.getMaster());
        scan.setRecordFilter(new RecordVariantFilter(newDep.id.getMaster(), varProps));
        LRepository repository = indexUpdateBuilder.getRepository();
        LTable table = repository.getTable(indexUpdateBuilder.getTable());
        final IdRecordScanner scanner = table.getScannerWithIds(scan);
        IdRecord next;
        while ((next = scanner.next()) != null) {
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
            } catch (Exception e) {
                throw new IOException("Error setting up LilyClient", e);
            }

            LRepository repository = lilyClient.getRepository(repositoryName);
            RecordScan scan = getScan(repository);

            // Determine start and stop row
            byte[] startRow;
            if (scan.getRawStartRecordId() != null) {
                startRow = scan.getRawStartRecordId();
            } else if (scan.getStartRecordId() != null) {
                startRow = scan.getStartRecordId().toBytes();
            } else {
                startRow = new byte[0];
            }

            byte[] stopRow;
            if (scan.getRawStopRecordId() != null) {
                stopRow = scan.getRawStopRecordId();
            } else if (scan.getStopRecordId() != null) {
                stopRow = scan.getStopRecordId().toBytes();
            } else {
                stopRow = new byte[0];
            }

            //
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
    protected boolean includeRegionInSplit(final byte[] startKey, final byte [] endKey) {
        return true;
    }

    protected RecordScan getScan(LRepository repository) {
        RecordScan scan;
        String scanData = conf.get(SCAN);
        if (scanData != null) {
            try {
                JsonNode node = JsonFormat.deserializeNonStd(scanData);
                scan = RecordScanReader.INSTANCE.fromJson(node, repository);
            } catch (Exception e) {
                ExceptionUtil.handleInterrupt(e);
                throw new RuntimeException(e);
            }
        } else {
            scan = new RecordScan();
        }
        return scan;
    }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        } catch (RepositoryException e) {
            throw new IOException("Error getting Lily repository object", e);
        }

        // Build RecordScan
        RecordScan scan = getScan(repository);

        // Change the start/stop record IDs on the scan to the current split
        TableSplit split = (TableSplit)inputSplit;
        scan.setRawStartRecordId(split.getStartRow());
        scan.setRawStopRecordId(split.getEndRow());

        RecordScanner scanner = null;
        try {
            String hbaseTableName = Bytes.toString(split.getTableName());
            String repositoryTableName = RepoAndTableUtil.extractLilyTableName(repositoryName, hbaseTableName);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        } catch (RepositoryException e) {
            throw new IOException("Error getting Lily repository object", e);
        }

        // Build RecordScan
        RecordScan scan = getScan(repository);

        // Change the start/stop record IDs on the scan to the current split
        TableSplit split = (TableSplit)inputSplit;
        scan.setRawStartRecordId(split.getStartRow());
        scan.setRawStopRecordId(split.getEndRow());

        IdRecordScanner scanner = null;
        try {
            String hbaseTableName = Bytes.toString(split.getTableName());
            String repositoryTableName = RepoAndTableUtil.extractLilyTableName(repositoryName, hbaseTableName);
View Full Code Here
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
            fieldValues.add(value);
            record.setField(fieldType1.getName(), value);
            repository.create(record);
        }

        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("ZA"));
        scan.setStopRecordId(idGenerator.newRecordId("ZZ")); // stop row is exclusive

        RecordScanner scanner = repository.getScanner(scan);

        int i = 0;
        Record record;
        while ((record = scanner.next()) != null) {
            assertEquals(record.getField(fieldType1.getName()), fieldValues.get(i));
            i++;
        }
        scanner.close();
        assertEquals("Found 25 records", 25, i);

        // Same using for-each loop
        scanner = repository.getScanner(scan);
        i = 0;
        for (Record result : scanner) {
            assertEquals(result.getField(fieldType1.getName()), fieldValues.get(i));
            i++;
        }
        scanner.close();
        assertEquals("Found 25 records", 25, i);

        // Scan all records, this should give at least 26 results
        scan = new RecordScan();
        scanner = repository.getScanner(scan);
        i = 0;
        while (scanner.next() != null) {
            i++;
        }
View Full Code Here
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
        Record record = repository.newRecord(id);
        record.setRecordType(recordType1.getName());
        record.setField(fieldType1.getName(), "dummy value");
        repository.create(record);

        RecordScan scan = new RecordScan();

        IdRecordScanner scanner = repository.getScannerWithIds(scan);
        final IdRecord next = scanner.next();
        assertNotNull(next);
        assertFalse(next.getFieldIdToNameMapping().isEmpty());
View Full Code Here
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
        Record record = repository.newRecord(id);
        record.setRecordType(recordType1.getName());
        record.setField(fieldType1.getName(), "dummy value");
        repository.create(record);

        RecordScan scan = new RecordScan();

        // normal Record scanner!
        RecordScanner scanner = repository.getScanner(scan);
        final Record next = scanner.next();
        assertNotNull(next);
View Full Code Here
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
        repository.recordBuilder().recordType(rt1.getName()).field(fieldType1.getName(), "value").create();
        repository.recordBuilder().recordType(rt1.getName()).field(fieldType1.getName(), "value").create();
        repository.recordBuilder().recordType(rt2.getName()).field(fieldType1.getName(), "value").create();
        repository.recordBuilder().recordType(rt2.getName(), 1L).field(fieldType1.getName(), "value").create();

        RecordScan scan = new RecordScan();
        scan.setRecordFilter(new RecordTypeFilter(rt1.getName()));
        assertEquals(2, countResults(repository.getScanner(scan)));

        scan = new RecordScan();
        scan.setRecordFilter(new RecordTypeFilter(rt2.getName()));
        assertEquals(2, countResults(repository.getScanner(scan)));

        scan = new RecordScan();
        scan.setRecordFilter(new RecordTypeFilter(rt2.getName(), 2L));
        assertEquals(1, countResults(repository.getScanner(scan)));
    }
View Full Code Here
TOP

Related Classes of org.lilyproject.repository.api.RecordScan

  • $.MyJob
  • org.lilyproject.indexer.model.indexerconf.IndexerUtils
  • org.lilyproject.mapreduce.AbstractLilyScanInputFormat
  • org.lilyproject.mapreduce.LilyIdScanInputFormat
  • org.lilyproject.mapreduce.LilyScanInputFormat
  • org.lilyproject.mapreduce.test.MapReduceTest
  • org.lilyproject.process.test.LilyClientTest
  • org.lilyproject.repository.impl.test.AbstractRepositoryTest
  • org.lilyproject.tools.import_.json.RecordScanReader
  • org.lilyproject.tools.import_.json.test.JsonConversionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.