Package org.lilyproject.repository.api

Examples of org.lilyproject.repository.api.RecordScan

136
137
138
139
140
141
142
143
144
145
146
147
148
            record.setField(new QName(NS, "scanfield"), "value " + i);
            repository.create(record);
        }

        // Do a scan
        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("A"));
        scan.setStopRecordId(idGenerator.newRecordId("B"));

        RecordScanner scanner = repository.getScanner(scan);
        int i = 0;
        while (scanner.next() != null) {
            i++;
View Full Code Here
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
                .field(fieldType1.getName(), "foo")
                .create();

        {
            // master scan
            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(new RecordVariantFilter(master, new HashMap<String, String>()));
            RecordScanner scanner = repository.getScanner(scan);
            assertEquals(master, scanner.next().getId());
            assertNull(scanner.next());
            scanner.close();
        }

        {
            // variant scan
            final HashMap<String, String> variantProperties = new HashMap<String, String>();
            variantProperties.put("foo", null);
            variantProperties.put("bar", null);

            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(new RecordVariantFilter(variant.getMaster(), variantProperties));
            RecordScanner scanner = repository.getScanner(scan);
            assertNull(scanner.next()); // it doesn't match anything
            scanner.close();
        }

        {
            // variant scan for something completely different
            final HashMap<String, String> variantProperties = new HashMap<String, String>();
            variantProperties.put("key1", null);
            variantProperties.put("key2", null);

            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(new RecordVariantFilter(variant.getMaster(), variantProperties));
            RecordScanner scanner = repository.getScanner(scan);
            assertEquals(variant, scanner.next().getId());
            assertEquals(variantWithOtherValues, scanner.next().getId());
            assertNull(scanner.next()); // it doesn't match the extendedVariant!
            scanner.close();
        }

        {
            // extended variant scan
            final HashMap<String, String> variantProperties = new HashMap<String, String>();
            variantProperties.put("key1", null);
            variantProperties.put("key2", null);
            variantProperties.put("key3", null);

            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(
                    new RecordVariantFilter(extendedVariant.getMaster(), variantProperties));
            RecordScanner scanner = repository.getScanner(scan);
            assertEquals(extendedVariant, scanner.next().getId());
            assertNull(scanner.next());
            scanner.close();
View Full Code Here
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
            // variant scan with all values
            final HashMap<String, String> variantProperties = new HashMap<String, String>();
            variantProperties.put("key1", "value1");
            variantProperties.put("key2", "value2");

            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(new RecordVariantFilter(variant.getMaster(), variantProperties));
            RecordScanner scanner = repository.getScanner(scan);
            assertEquals(variant, scanner.next().getId());
            assertNull(scanner.next()); // it doesn't match the other variants
            scanner.close();
        }

        {
            // variant scan with specific value for key2 only
            final HashMap<String, String> variantProperties = new HashMap<String, String>();
            variantProperties.put("key1", null);
            variantProperties.put("key2", "value2");

            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(new RecordVariantFilter(variant.getMaster(), variantProperties));
            RecordScanner scanner = repository.getScanner(scan);
            assertEquals(variant, scanner.next().getId());
            assertEquals(variantWithOtherValues, scanner.next().getId()); // has same value for key2
            assertNull(scanner.next()); // it doesn't match the other variants
            scanner.close();
        }

        {
            // variant scan with specific value for key1 only
            final HashMap<String, String> variantProperties = new HashMap<String, String>();
            variantProperties.put("key1", "value1");
            variantProperties.put("key2", null);

            final RecordScan scan = new RecordScan();
            scan.setRecordFilter(new RecordVariantFilter(variant.getMaster(), variantProperties));
            RecordScanner scanner = repository.getScanner(scan);
            assertEquals(variant, scanner.next().getId());
            assertNull(scanner.next()); // it doesn't match the other variants
            scanner.close();
        }
View Full Code Here
3634
3635
3636
3637
3638
3639
3640
3641
3642
        record.setRecordType(recordType1.getName());
        record.setField(fieldType1.getName(), "stop drinking coke");
        record.setMetadata(fieldType1.getName(), new MetadataBuilder().value("field1", "foobar").build());
        repository.create(record);

        RecordScan scan = new RecordScan();
        scan.setRecordFilter(new FieldValueFilter(fieldType1.getName(), "stop drinking coke"));
        assertEquals(1, countResults(repository.getScanner(scan)));
    }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
256
257
            job.setOutputFormatClass(NullOutputFormat.class);

            job.setNumReduceTasks(0);

            RecordScan scan = new RecordScan();
            scan.setStartRecordId(idGenerator.newRecordId(String.format("%1$03d", 15)));
            scan.setStopRecordId(idGenerator.newRecordId(String.format("%1$03d", 25)));

            LilyMapReduceUtil.initMapperJob(scan, "localhost", repository, job);

            boolean b = job.waitForCompletion(true);
            if (!b) {
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
        this.table = table;
    }

    public void count(String startId, String stopId, String recordTypeFilter, File configFile) throws Exception {
        System.out.println("Counting records");
        RecordScan scan = createRecordScan(startId, stopId, recordTypeFilter, configFile);
        if (configFile == null) {
            // We don't need to return fields for counting
            scan.setReturnFields(ReturnFields.NONE);
        }
        RecordScanner scanner = table.getScanner(scan);
        Record record;
        Date start = new Date();
        try {
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
    public void print(String startId, String stopId, long limit, String recordTypeFilter, File configFile) throws Exception {
        if (limit < 0) {
            limit = Long.MAX_VALUE;
        }

        RecordScan scan = createRecordScan(startId, stopId, recordTypeFilter, configFile);
        RecordScanner scanner = table.getScanner(scan);
        try {
            int i = 0;
            Record record;
            while ((record = scanner.next()) != null && i < limit) {
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
        }

    }

    private RecordScan createRecordScan(String startId, String stopId, String recordTypeFilter, File scanConfFile) throws Exception {
        RecordScan scan = null;

        if (scanConfFile != null) {
            JsonNode jsonNode = JsonFormat.deserializeNonStd(new FileInputStream(scanConfFile));
            scan = RecordScanReader.INSTANCE.fromJson(jsonNode, repository);
        }

        scan = scan != null ? scan : new RecordScan();
        scan.setCaching(RecordScanTool.DEFAULT_CACHE);
        scan.setCacheBlocks(DEFAULT_CACHE_BLOCKS);

        if (startId != null && startId.length() > 0) {
            scan.setStartRecordId(repository.getIdGenerator().fromString(startId));
        }

        if (stopId != null && stopId.length() > 0) {
            scan.setStopRecordId(repository.getIdGenerator().fromString(stopId));
        }

        if (recordTypeFilter != null && !recordTypeFilter.isEmpty()) {
            RecordFilterList filterList = new RecordFilterList(Operator.MUST_PASS_ONE);
            if (scan.getRecordFilter() != null) {
                filterList.addFilter(scan.getRecordFilter());
            }
            scan.setRecordFilter(filterList);

            String[] recordTypes = recordTypeFilter.split(",");
            for (String recordType : recordTypes) {
                filterList.addFilter(new RecordTypeFilter(QName.fromString(recordType)));
            }
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    @Test
    public void testScanRecordId() throws Exception {
        IdGenerator idGenerator = new IdGeneratorImpl();

        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId());
        scan.setStopRecordId(idGenerator.newRecordId("foo"));

        byte[] data = scanToBytes(scan);
        RecordScan parsedScan = scanFromBytes(data);

        assertEquals(scan.getStartRecordId(), parsedScan.getStartRecordId());
        assertEquals(scan.getStopRecordId(), parsedScan.getStopRecordId());

        // RecordId's should be simply string properties in json
        JsonNode node = new ObjectMapper().readTree(data);
        assertEquals(scan.getStartRecordId().toString(), node.get("startRecordId").getTextValue());
        assertEquals("USER.foo", node.get("stopRecordId").getTextValue());
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
        scanFromBytes(json.getBytes());
    }

    @Test
    public void testScanRawStartStop() throws Exception {
        RecordScan scan = new RecordScan();
        scan.setRawStartRecordId(Bytes.toBytes("bar"));
        scan.setRawStopRecordId(Bytes.toBytes("foo"));

        byte[] data = scanToBytes(scan);
        RecordScan parsedScan = scanFromBytes(data);

        assertArrayEquals(scan.getRawStartRecordId(), parsedScan.getRawStartRecordId());
        assertArrayEquals(scan.getRawStopRecordId(), parsedScan.getRawStopRecordId());

        // Verify how the bytes are stored in the json
        JsonNode node = new ObjectMapper().readTree(data);
        assertEquals("YmFy", node.get("rawStartRecordId").getTextValue());
        assertEquals("Zm9v", node.get("rawStopRecordId").getTextValue());
View Full Code Here
TOP

Related Classes of org.lilyproject.repository.api.RecordScan

  • $.MyJob
  • org.lilyproject.indexer.model.indexerconf.IndexerUtils
  • org.lilyproject.mapreduce.AbstractLilyScanInputFormat
  • org.lilyproject.mapreduce.LilyIdScanInputFormat
  • org.lilyproject.mapreduce.LilyScanInputFormat
  • org.lilyproject.mapreduce.test.MapReduceTest
  • org.lilyproject.process.test.LilyClientTest
  • org.lilyproject.repository.impl.test.AbstractRepositoryTest
  • org.lilyproject.tools.import_.json.RecordScanReader
  • org.lilyproject.tools.import_.json.test.JsonConversionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.