Examples of org.lilyproject.util.repo.RecordEvent$IndexRecordFilterData

org.lilyproject.util.repo.RecordEvent
Data needed for IndexRecordFilter evaluation.
Information needed to decide if an IndexRecordFilter matches a record. Contains both the necessary information both from the old and new record state, so that we know what matched before and what matches now, which enables important optimisations.

For example, this information is used by the IndexerEditFilter to only sent events to sep consumers from relevant indexes, as well as by IndexUpdater to know what index inclusion rule matches before & now.

At the time of this writing, the indexerconf only allows selection based on record type, on 1 field, and on information that is part of the record id. The model here is already a bit more flexible (can contain info on multiple fields) to allow for more powerful selections in the future.

    }


    @Test
    public void testRecordEvent_JsonRoundTrip() throws Exception {


        RecordEvent event = new RecordEvent();
        byte[] json = event.toJsonBytes();
        event = new RecordEvent(json, idGenerator);


        assertNull(event.getIndexRecordFilterData());


        SchemaId oldRtId = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId newRtId = idGenerator.getSchemaId(UUID.randomUUID());


        RecordEvent.IndexRecordFilterData idxSel = new RecordEvent.IndexRecordFilterData();
        event.setIndexRecordFilterData(idxSel);
        idxSel.setOldRecordType(oldRtId);
        idxSel.setNewRecordType(newRtId);


        json = event.toJsonBytes();
        event = new RecordEvent(json, idGenerator);


        assertNotNull(event.getIndexRecordFilterData());
        assertEquals(oldRtId, event.getIndexRecordFilterData().getOldRecordType());
        assertEquals(newRtId, event.getIndexRecordFilterData().getNewRecordType());
        assertNull(event.getIndexRecordFilterData().getFieldChanges());


        SchemaId field1Id = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId field2Id = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId field3Id = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId field4Id = idGenerator.getSchemaId(UUID.randomUUID());


        event = new RecordEvent();
        idxSel = new RecordEvent.IndexRecordFilterData();
        event.setIndexRecordFilterData(idxSel);
        idxSel.addChangedField(field1Id, null, null);
        idxSel.addChangedField(field2Id, Bytes.toBytes("foo1"), Bytes.toBytes("foo2"));
        idxSel.addChangedField(field3Id, Bytes.toBytes("foo3"), null);
        idxSel.addChangedField(field4Id, null, Bytes.toBytes("foo4"));


        json = event.toJsonBytes();
        event = new RecordEvent(json, idGenerator);


        List<RecordEvent.FieldChange> fieldChanges = event.getIndexRecordFilterData().getFieldChanges();
        assertEquals(4, fieldChanges.size());


        assertEquals(field1Id, fieldChanges.get(0).getId());
        assertNull(fieldChanges.get(0).getOldValue());
        assertNull(fieldChanges.get(0).getNewValue());

View Full Code Here

    }


    @Test
    public void testRecordEvent_JsonRoundtrip_TableName() throws IOException {
        final String tableName = "_table_name_";
        RecordEvent recordEvent = new RecordEvent();
        recordEvent.setTableName(tableName);


        byte[] jsonBytes = recordEvent.toJsonBytes();


        RecordEvent deserialized = new RecordEvent(jsonBytes, idGenerator);


        assertEquals(tableName, deserialized.getTableName());
    }

View Full Code Here


        assertFalse(filterData.appliesToSubscription("not_included"));
    }


    private IndexRecordFilterData doJsonRoundtrip(IndexRecordFilterData recordFilterData) {
        RecordEvent recordEvent = new RecordEvent();
        recordEvent.setIndexRecordFilterData(recordFilterData);
        RecordEvent deserializedEvent;
        try {
            deserializedEvent = new RecordEvent(recordEvent.toJsonBytes(), idGenerator);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return deserializedEvent.getIndexRecordFilterData();
    }

View Full Code Here

                        // timestamps, and leave the non-versioned fields untouched.
                        clearData(recordId, null, Bytes.toLong(oldVersion));
                    }
                }


                RecordEvent recordEvent = new RecordEvent();
                recordEvent.setType(Type.CREATE);
                recordEvent.setTableName(getTableName());
                if (record.hasAttributes()) {
                    recordEvent.getAttributes().putAll(record.getAttributes());
                }


                Record newRecord = record.cloneRecord();
                newRecord.setId(recordId);


                for (RecordUpdateHook hook : updateHooks) {
                    hook.beforeCreate(newRecord, this, fieldTypes, recordEvent);
                }


                Set<BlobReference> referencedBlobs = new HashSet<BlobReference>();
                Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>();


                Put put = buildPut(newRecord, version, fieldTypes, recordEvent, referencedBlobs, unReferencedBlobs, newOcc);


                // Make sure the record type changed flag stays false for a newly
                // created record
                recordEvent.setRecordTypeChanged(false);
                Long newVersion = newRecord.getVersion();
                if (newVersion != null) {
                    recordEvent.setVersionCreated(newVersion);
                }


                // Reserve blobs so no other records can use them
                reserveBlobs(null, referencedBlobs);


                put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes());
                boolean success = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes,
                        oldOccBytes, put);
                if (!success) {
                    throw new RecordExistsException(recordId);
                }

View Full Code Here

            Pair<Record, byte[]> recordAndOcc = readWithOcc(record.getId(), null, null, fieldTypes);
            Record originalRecord = new UnmodifiableRecord(recordAndOcc.getV1());


            byte[] oldOccBytes = recordAndOcc.getV2();


            RecordEvent recordEvent = new RecordEvent();
            recordEvent.setType(Type.UPDATE);
            recordEvent.setTableName(getTableName());
            if (record.hasAttributes()) {
                recordEvent.getAttributes().putAll(record.getAttributes());
            }


            for (RecordUpdateHook hook : updateHooks) {
                hook.beforeUpdate(record, originalRecord, this, fieldTypes, recordEvent);
            }


            Record newRecord = record.cloneRecord();


            Put put = new Put(newRecord.getId().toBytes());
            Set<BlobReference> referencedBlobs = new HashSet<BlobReference>();
            Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>();
            long newVersion = originalRecord.getVersion() == null ? 1 : originalRecord.getVersion() + 1;


            // Check the mutation conditions.
            // It is important that we do this before checking if the record needs updating at all: otherwise,
            // another client might already have performed the update we intended to do, which is problematic
            // in cases like incrementing a counter (the counter should be updated twice, not once).
            Record conditionsResponse = MutationConditionVerifier.checkConditions(originalRecord, conditions, this,
                    record);
            if (conditionsResponse != null) {
                return conditionsResponse;
            }


            if (calculateRecordChanges(newRecord, originalRecord, newVersion, put, recordEvent, referencedBlobs,
                    unReferencedBlobs, useLatestRecordType, fieldTypes)) {


                // Reserve blobs so no other records can use them
                reserveBlobs(record.getId(), referencedBlobs);


                put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes());
                put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOccBytes));
                boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes,
                        oldOccBytes, put);
                if (!occSuccess) {
                    throw new ConcurrentRecordUpdateException(recordId);

View Full Code Here


            Put put = new Put(recordId.toBytes());
            Set<BlobReference> referencedBlobs = new HashSet<BlobReference>();
            Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>();


            RecordEvent recordEvent = new RecordEvent();
            recordEvent.setType(Type.UPDATE);
            recordEvent.setTableName(getTableName());
            recordEvent.setVersionUpdated(version);


            for (RecordUpdateHook hook : updateHooks) {
                hook.beforeUpdate(record, originalRecord, this, fieldTypes, recordEvent);
            }


            Set<Scope> changedScopes = calculateUpdateFields(record, fields, record.getMetadataMap(), originalFields,
                    originalRecord.getMetadataMap(), originalNextFields, version, put,
                    recordEvent, referencedBlobs, unReferencedBlobs, true, fieldTypes);
            for (BlobReference referencedBlob : referencedBlobs) {
                referencedBlob.setRecordId(recordId);
            }
            for (BlobReference unReferencedBlob : unReferencedBlobs) {
                unReferencedBlob.setRecordId(recordId);
            }


            if (!changedScopes.isEmpty()) {
                // Check the conditions after establishing that the record really needs updating, this makes the
                // conditional update operation idempotent.
                Record conditionsRecord = MutationConditionVerifier.checkConditions(originalRecord, conditions, this,
                        record);
                if (conditionsRecord != null) {
                    return conditionsRecord;
                }


                // Update the record types


                // If no record type is specified explicitly, use the current one of the non-versioned scope
                QName recordTypeName = record.getRecordTypeName() != null ? record.getRecordTypeName() :
                        originalRecord.getRecordTypeName();
                Long recordTypeVersion;
                if (latestRecordType) {
                    recordTypeVersion = null;
                } else if (record.getRecordTypeName() == null) {
                    recordTypeVersion = originalRecord.getRecordTypeVersion();
                } else {
                    recordTypeVersion = record.getRecordTypeVersion();
                }
                RecordType recordType = typeManager.getRecordTypeByName(recordTypeName, recordTypeVersion);


                // Update the mutable record type in the record object
                Scope mutableScope = Scope.VERSIONED_MUTABLE;
                newRecord.setRecordType(mutableScope, recordType.getName(), recordType.getVersion());


                // If the record type changed, update it on the record table
                QName originalMutableScopeRecordTypeName = originalRecord.getRecordTypeName(mutableScope);
                if (originalMutableScopeRecordTypeName == null) { // There was no initial mutable record type yet
                    put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version,
                            recordType.getId().getBytes());
                    put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(mutableScope), version,
                            Bytes.toBytes(recordType.getVersion()));
                } else {
                    RecordType originalMutableScopeRecordType = typeManager
                            .getRecordTypeByName(originalMutableScopeRecordTypeName,
                                    originalRecord.getRecordTypeVersion(mutableScope));
                    if (!recordType.getId().equals(originalMutableScopeRecordType.getId())) {
                        // If the next record version had the same record type name, copy the original value to that one
                        if (originalNextRecord != null && originalMutableScopeRecordType.getName()
                                .equals(originalNextRecord.getRecordTypeName(mutableScope))) {
                            put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version + 1,
                                    originalMutableScopeRecordType.getId().getBytes());
                        }
                        put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version,
                                recordType.getId().getBytes());
                    }
                    if (!recordType.getVersion().equals(originalMutableScopeRecordType.getVersion())) {
                        // If the next record version had the same record type version, copy the original value to that one
                        if (originalNextRecord != null && originalMutableScopeRecordType.getVersion()
                                .equals(originalNextRecord.getRecordTypeVersion(mutableScope))) {
                            put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version + 1,
                                    Bytes.toBytes(originalMutableScopeRecordType.getVersion()));
                        }
                        put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(mutableScope), version,
                                Bytes.toBytes(recordType.getVersion()));
                    }
                }


                // Validate if the new values for the record are valid wrt the recordType (e.g. mandatory fields)
                validateRecord(newRecord, originalRecord, recordType, fieldTypes);


                // Reserve blobs so no other records can use them
                reserveBlobs(record.getId(), referencedBlobs);


                put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, 1L, recordEvent.toJsonBytes());
                put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOccBytes));
                boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes,
                        oldOccBytes, put);
                if (!occSuccess) {
                    throw new ConcurrentRecordUpdateException(recordId);

View Full Code Here

                }
            }


            byte[] oldOcc = recordAndOcc.getV2();


            RecordEvent recordEvent = new RecordEvent();
            recordEvent.setType(Type.DELETE);
            recordEvent.setTableName(getTableName());
            if (attributes != null && !attributes.isEmpty()) {
                recordEvent.getAttributes().putAll(attributes);
            }


            for (RecordUpdateHook hook : updateHooks) {
                hook.beforeDelete(originalRecord, this, fieldTypes, recordEvent);
            }


            if (conditions != null) {
                Record conditionsRecord = MutationConditionVerifier.checkConditions(originalRecord, conditions, this,
                        null);
                if (conditionsRecord != null) {
                    return conditionsRecord;
                }
            }


            Put put = new Put(rowId);
            // Mark the record as deleted
            put.add(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes, 1L, Bytes.toBytes(true));


            // Put the delete marker in the non-versioned fields instead of deleting their columns in the clearData call
            // This is needed to avoid non-versioned fields to be lost due to the hbase delete thombstone
            // See trac ticket http://dev.outerthought.org/trac/outerthought_lilyproject/ticket/297
            Map<QName, Object> fields = originalRecord.getFields();
            for (Entry<QName, Object> fieldEntry : fields.entrySet()) {
                FieldTypeImpl fieldType = (FieldTypeImpl) fieldTypes.getFieldType(fieldEntry.getKey());
                if (Scope.NON_VERSIONED == fieldType.getScope()) {
                    put.add(RecordCf.DATA.bytes, fieldType.getQualifier(), 1L, FieldFlags.getDeleteMarker());
                }


            }


            put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes());
            put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOcc));


            // Hint towards the NGDATA HBase authorization coprocessor: for deletes, we need write access to all
            // columns, since otherwise we could end up with half-deleted records. The default behavior for puts
            // is to silently filter columns from the Put for which the user has no write permission.

View Full Code Here

        expectEvent(type, table, recordId, versionCreated, versionUpdated, false, updatedFields);
    }


    private void expectEvent(RecordEvent.Type type, String table, RecordId recordId, Long versionCreated, Long versionUpdated,
                             boolean recordTypeChanged, SchemaId... updatedFields) {
        RecordEvent event = new RecordEvent();


        event.setType(type);
        event.setTableName(table);


        for (SchemaId updatedField : updatedFields) {
            event.addUpdatedField(updatedField);
        }


        if (versionCreated != null) {
            event.setVersionCreated(versionCreated);
        }


        if (versionUpdated != null) {
            event.setVersionUpdated(versionUpdated);
        }


        if (recordTypeChanged) {
            event.setRecordTypeChanged(recordTypeChanged);
        }


        messageVerifier.addExpectedEvent(recordId, event);
    }

View Full Code Here

            // test that the failures variable is 0.


            RecordId recordId = repository.getIdGenerator().fromBytes(event.getRow());


            try {
                RecordEvent recordEvent = new RecordEvent(event.getPayload(), idGenerator);


                if (recordEvent.getType().equals(RecordEvent.Type.INDEX)) {
                    log.debug("Ignoring incoming re-index event for message verification");
                    return;
                }


                if (expectedEvents.isEmpty()) {
                    System.err
                            .println("No events are expected, but we just got event " + recordEvent.toJson() + " on " + recordId);
                    failures++;
                    return;
                }


                Pair<RecordId, RecordEvent> expectedPair = expectedEvents.remove(0);
                RecordId expectedId = expectedPair.getV1();
                RecordEvent expectedEvent = expectedPair.getV2();


                if (expectedEvent == null) {
                    failures++;
                    printSomethingLoad();
                    System.err.println("Did not expect a message, but got:");
                    System.err.println(recordId);
                    System.err.println(recordEvent.toJson());
                } else {
                    if (!recordEvent.equals(expectedEvent) ||
                            !(recordId.equals(expectedId) ||
                                    (expectedId == null && expectedEvent.getType() == CREATE))) {
                        failures++;
                        printSomethingLoad();
                        System.err.println("Expected message:");
                        System.err.println(expectedId);
                        System.err.println(expectedEvent.toJson());
                        System.err.println("Received message:");
                        System.err.println(recordId);
                        System.err.println(recordEvent.toJson());
                    } else {
                        log.debug("Received message ok.");

View Full Code Here

            throw new RuntimeException(e);
        } catch (RepositoryException e) {
            throw new RuntimeException(e);
        }


        RecordEvent recordEvent;
        try {
            recordEvent = event.getRecordEvent();
        } catch (IOException e) {
            log.error("Error reading record event, processing of message cancelled", e);
            return;

View Full Code Here

0 1 2

TOP

Related Classes of org.lilyproject.util.repo.RecordEvent$IndexRecordFilterData

org.codehaus.jackson.JsonGenerator

org.codehaus.jackson.JsonParser

org.codehaus.jackson.JsonToken

org.codehaus.jackson.util.ByteArrayBuilder

org.lilyproject.indexer.engine.test.IndexerTest

org.lilyproject.indexer.engine.test.IndexerTest$MessageVerifier

org.lilyproject.indexer.event.IndexerEditFilter

org.lilyproject.indexer.event.IndexerEditFilterTest

org.lilyproject.indexer.hbase.mapper.LilyResultToSolrMapper

org.lilyproject.indexer.hbase.mapper.LilyResultToSolrMapperTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.