Examples of org.lilyproject.util.repo.RecordEvent

org.lilyproject.util.repo.RecordEvent
Represents the payload of an event about a create-update-delete operation on the repository.
The actual payload is json, this class helps in parsing or constructing that json.

    }


    @Test
    public void testRecordEvent_JsonRoundTrip() throws Exception {


        RecordEvent event = new RecordEvent();
        byte[] json = event.toJsonBytes();
        event = new RecordEvent(json, idGenerator);


        assertNull(event.getIndexRecordFilterData());


        SchemaId oldRtId = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId newRtId = idGenerator.getSchemaId(UUID.randomUUID());


        RecordEvent.IndexRecordFilterData idxSel = new RecordEvent.IndexRecordFilterData();
        event.setIndexRecordFilterData(idxSel);
        idxSel.setOldRecordType(oldRtId);
        idxSel.setNewRecordType(newRtId);


        json = event.toJsonBytes();
        event = new RecordEvent(json, idGenerator);


        assertNotNull(event.getIndexRecordFilterData());
        assertEquals(oldRtId, event.getIndexRecordFilterData().getOldRecordType());
        assertEquals(newRtId, event.getIndexRecordFilterData().getNewRecordType());
        assertNull(event.getIndexRecordFilterData().getFieldChanges());


        SchemaId field1Id = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId field2Id = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId field3Id = idGenerator.getSchemaId(UUID.randomUUID());
        SchemaId field4Id = idGenerator.getSchemaId(UUID.randomUUID());


        event = new RecordEvent();
        idxSel = new RecordEvent.IndexRecordFilterData();
        event.setIndexRecordFilterData(idxSel);
        idxSel.addChangedField(field1Id, null, null);
        idxSel.addChangedField(field2Id, Bytes.toBytes("foo1"), Bytes.toBytes("foo2"));
        idxSel.addChangedField(field3Id, Bytes.toBytes("foo3"), null);
        idxSel.addChangedField(field4Id, null, Bytes.toBytes("foo4"));


        json = event.toJsonBytes();
        event = new RecordEvent(json, idGenerator);


        List<RecordEvent.FieldChange> fieldChanges = event.getIndexRecordFilterData().getFieldChanges();
        assertEquals(4, fieldChanges.size());


        assertEquals(field1Id, fieldChanges.get(0).getId());
        assertNull(fieldChanges.get(0).getOldValue());
        assertNull(fieldChanges.get(0).getNewValue());

View Full Code Here

    }


    @Test
    public void testRecordEvent_JsonRoundtrip_TableName() throws IOException {
        final String tableName = "_table_name_";
        RecordEvent recordEvent = new RecordEvent();
        recordEvent.setTableName(tableName);


        byte[] jsonBytes = recordEvent.toJsonBytes();


        RecordEvent deserialized = new RecordEvent(jsonBytes, idGenerator);


        assertEquals(tableName, deserialized.getTableName());
    }

View Full Code Here


        assertFalse(filterData.appliesToSubscription("not_included"));
    }


    private IndexRecordFilterData doJsonRoundtrip(IndexRecordFilterData recordFilterData) {
        RecordEvent recordEvent = new RecordEvent();
        recordEvent.setIndexRecordFilterData(recordFilterData);
        RecordEvent deserializedEvent;
        try {
            deserializedEvent = new RecordEvent(recordEvent.toJsonBytes(), idGenerator);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return deserializedEvent.getIndexRecordFilterData();
    }

View Full Code Here

                        // timestamps, and leave the non-versioned fields untouched.
                        clearData(recordId, null, Bytes.toLong(oldVersion));
                    }
                }


                RecordEvent recordEvent = new RecordEvent();
                recordEvent.setType(Type.CREATE);
                recordEvent.setTableName(getTableName());
                if (record.hasAttributes()) {
                    recordEvent.getAttributes().putAll(record.getAttributes());
                }


                Record newRecord = record.cloneRecord();
                newRecord.setId(recordId);


                for (RecordUpdateHook hook : updateHooks) {
                    hook.beforeCreate(newRecord, this, fieldTypes, recordEvent);
                }


                Set<BlobReference> referencedBlobs = new HashSet<BlobReference>();
                Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>();


                Put put = buildPut(newRecord, version, fieldTypes, recordEvent, referencedBlobs, unReferencedBlobs, newOcc);


                // Make sure the record type changed flag stays false for a newly
                // created record
                recordEvent.setRecordTypeChanged(false);
                Long newVersion = newRecord.getVersion();
                if (newVersion != null) {
                    recordEvent.setVersionCreated(newVersion);
                }


                // Reserve blobs so no other records can use them
                reserveBlobs(null, referencedBlobs);


                put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes());
                boolean success = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes,
                        oldOccBytes, put);
                if (!success) {
                    throw new RecordExistsException(recordId);
                }

View Full Code Here

            Pair<Record, byte[]> recordAndOcc = readWithOcc(record.getId(), null, null, fieldTypes);
            Record originalRecord = new UnmodifiableRecord(recordAndOcc.getV1());


            byte[] oldOccBytes = recordAndOcc.getV2();


            RecordEvent recordEvent = new RecordEvent();
            recordEvent.setType(Type.UPDATE);
            recordEvent.setTableName(getTableName());
            if (record.hasAttributes()) {
                recordEvent.getAttributes().putAll(record.getAttributes());
            }


            for (RecordUpdateHook hook : updateHooks) {
                hook.beforeUpdate(record, originalRecord, this, fieldTypes, recordEvent);
            }


            Record newRecord = record.cloneRecord();


            Put put = new Put(newRecord.getId().toBytes());
            Set<BlobReference> referencedBlobs = new HashSet<BlobReference>();
            Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>();
            long newVersion = originalRecord.getVersion() == null ? 1 : originalRecord.getVersion() + 1;


            // Check the mutation conditions.
            // It is important that we do this before checking if the record needs updating at all: otherwise,
            // another client might already have performed the update we intended to do, which is problematic
            // in cases like incrementing a counter (the counter should be updated twice, not once).
            Record conditionsResponse = MutationConditionVerifier.checkConditions(originalRecord, conditions, this,
                    record);
            if (conditionsResponse != null) {
                return conditionsResponse;
            }


            if (calculateRecordChanges(newRecord, originalRecord, newVersion, put, recordEvent, referencedBlobs,
                    unReferencedBlobs, useLatestRecordType, fieldTypes)) {


                // Reserve blobs so no other records can use them
                reserveBlobs(record.getId(), referencedBlobs);


                put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes());
                put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOccBytes));
                boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes,
                        oldOccBytes, put);
                if (!occSuccess) {
                    throw new ConcurrentRecordUpdateException(recordId);

View Full Code Here


            Put put = new Put(recordId.toBytes());
            Set<BlobReference> referencedBlobs = new HashSet<BlobReference>();
            Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>();


            RecordEvent recordEvent = new RecordEvent();
            recordEvent.setType(Type.UPDATE);
            recordEvent.setTableName(getTableName());
            recordEvent.setVersionUpdated(version);


            for (RecordUpdateHook hook : updateHooks) {
                hook.beforeUpdate(record, originalRecord, this, fieldTypes, recordEvent);
            }


            Set<Scope> changedScopes = calculateUpdateFields(record, fields, record.getMetadataMap(), originalFields,
                    originalRecord.getMetadataMap(), originalNextFields, version, put,
                    recordEvent, referencedBlobs, unReferencedBlobs, true, fieldTypes);
            for (BlobReference referencedBlob : referencedBlobs) {
                referencedBlob.setRecordId(recordId);
            }
            for (BlobReference unReferencedBlob : unReferencedBlobs) {
                unReferencedBlob.setRecordId(recordId);
            }


            if (!changedScopes.isEmpty()) {
                // Check the conditions after establishing that the record really needs updating, this makes the
                // conditional update operation idempotent.
                Record conditionsRecord = MutationConditionVerifier.checkConditions(originalRecord, conditions, this,
                        record);
                if (conditionsRecord != null) {
                    return conditionsRecord;
                }


                // Update the record types


                // If no record type is specified explicitly, use the current one of the non-versioned scope
                QName recordTypeName = record.getRecordTypeName() != null ? record.getRecordTypeName() :
                        originalRecord.getRecordTypeName();
                Long recordTypeVersion;
                if (latestRecordType) {
                    recordTypeVersion = null;
                } else if (record.getRecordTypeName() == null) {
                    recordTypeVersion = originalRecord.getRecordTypeVersion();
                } else {
                    recordTypeVersion = record.getRecordTypeVersion();
                }
                RecordType recordType = typeManager.getRecordTypeByName(recordTypeName, recordTypeVersion);


                // Update the mutable record type in the record object
                Scope mutableScope = Scope.VERSIONED_MUTABLE;
                newRecord.setRecordType(mutableScope, recordType.getName(), recordType.getVersion());


                // If the record type changed, update it on the record table
                QName originalMutableScopeRecordTypeName = originalRecord.getRecordTypeName(mutableScope);
                if (originalMutableScopeRecordTypeName == null) { // There was no initial mutable record type yet
                    put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version,
                            recordType.getId().getBytes());
                    put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(mutableScope), version,
                            Bytes.toBytes(recordType.getVersion()));
                } else {
                    RecordType originalMutableScopeRecordType = typeManager
                            .getRecordTypeByName(originalMutableScopeRecordTypeName,
                                    originalRecord.getRecordTypeVersion(mutableScope));
                    if (!recordType.getId().equals(originalMutableScopeRecordType.getId())) {
                        // If the next record version had the same record type name, copy the original value to that one
                        if (originalNextRecord != null && originalMutableScopeRecordType.getName()
                                .equals(originalNextRecord.getRecordTypeName(mutableScope))) {
                            put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version + 1,
                                    originalMutableScopeRecordType.getId().getBytes());
                        }
                        put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version,
                                recordType.getId().getBytes());
                    }
                    if (!recordType.getVersion().equals(originalMutableScopeRecordType.getVersion())) {
                        // If the next record version had the same record type version, copy the original value to that one
                        if (originalNextRecord != null && originalMutableScopeRecordType.getVersion()
                                .equals(originalNextRecord.getRecordTypeVersion(mutableScope))) {
                            put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version + 1,
                                    Bytes.toBytes(originalMutableScopeRecordType.getVersion()));
                        }
                        put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(mutableScope), version,
                                Bytes.toBytes(recordType.getVersion()));
                    }
                }


                // Validate if the new values for the record are valid wrt the recordType (e.g. mandatory fields)
                validateRecord(newRecord, originalRecord, recordType, fieldTypes);


                // Reserve blobs so no other records can use them
                reserveBlobs(record.getId(), referencedBlobs);


                put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, 1L, recordEvent.toJsonBytes());
                put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOccBytes));
                boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes,
                        oldOccBytes, put);
                if (!occSuccess) {
                    throw new ConcurrentRecordUpdateException(recordId);

View Full Code Here

                }
            }


            byte[] oldOcc = recordAndOcc.getV2();


            RecordEvent recordEvent = new RecordEvent();
            recordEvent.setType(Type.DELETE);
            recordEvent.setTableName(getTableName());
            if (attributes != null && !attributes.isEmpty()) {
                recordEvent.getAttributes().putAll(attributes);
            }


            for (RecordUpdateHook hook : updateHooks) {
                hook.beforeDelete(originalRecord, this, fieldTypes, recordEvent);
            }


            if (conditions != null) {
                Record conditionsRecord = MutationConditionVerifier.checkConditions(originalRecord, conditions, this,
                        null);
                if (conditionsRecord != null) {
                    return conditionsRecord;
                }
            }


            Put put = new Put(rowId);
            // Mark the record as deleted
            put.add(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes, 1L, Bytes.toBytes(true));


            // Put the delete marker in the non-versioned fields instead of deleting their columns in the clearData call
            // This is needed to avoid non-versioned fields to be lost due to the hbase delete thombstone
            // See trac ticket http://dev.outerthought.org/trac/outerthought_lilyproject/ticket/297
            Map<QName, Object> fields = originalRecord.getFields();
            for (Entry<QName, Object> fieldEntry : fields.entrySet()) {
                FieldTypeImpl fieldType = (FieldTypeImpl) fieldTypes.getFieldType(fieldEntry.getKey());
                if (Scope.NON_VERSIONED == fieldType.getScope()) {
                    put.add(RecordCf.DATA.bytes, fieldType.getQualifier(), 1L, FieldFlags.getDeleteMarker());
                }


            }


            put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes());
            put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOcc));


            // Hint towards the NGDATA HBase authorization coprocessor: for deletes, we need write access to all
            // columns, since otherwise we could end up with half-deleted records. The default behavior for puts
            // is to silently filter columns from the Put for which the user has no write permission.

View Full Code Here

        expectEvent(type, table, recordId, versionCreated, versionUpdated, false, updatedFields);
    }


    private void expectEvent(RecordEvent.Type type, String table, RecordId recordId, Long versionCreated, Long versionUpdated,
                             boolean recordTypeChanged, SchemaId... updatedFields) {
        RecordEvent event = new RecordEvent();


        event.setType(type);
        event.setTableName(table);


        for (SchemaId updatedField : updatedFields) {
            event.addUpdatedField(updatedField);
        }


        if (versionCreated != null) {
            event.setVersionCreated(versionCreated);
        }


        if (versionUpdated != null) {
            event.setVersionUpdated(versionUpdated);
        }


        if (recordTypeChanged) {
            event.setRecordTypeChanged(recordTypeChanged);
        }


        messageVerifier.addExpectedEvent(recordId, event);
    }

View Full Code Here

            // test that the failures variable is 0.


            RecordId recordId = repository.getIdGenerator().fromBytes(event.getRow());


            try {
                RecordEvent recordEvent = new RecordEvent(event.getPayload(), idGenerator);


                if (recordEvent.getType().equals(RecordEvent.Type.INDEX)) {
                    log.debug("Ignoring incoming re-index event for message verification");
                    return;
                }


                if (expectedEvents.isEmpty()) {
                    System.err
                            .println("No events are expected, but we just got event " + recordEvent.toJson() + " on " + recordId);
                    failures++;
                    return;
                }


                Pair<RecordId, RecordEvent> expectedPair = expectedEvents.remove(0);
                RecordId expectedId = expectedPair.getV1();
                RecordEvent expectedEvent = expectedPair.getV2();


                if (expectedEvent == null) {
                    failures++;
                    printSomethingLoad();
                    System.err.println("Did not expect a message, but got:");
                    System.err.println(recordId);
                    System.err.println(recordEvent.toJson());
                } else {
                    if (!recordEvent.equals(expectedEvent) ||
                            !(recordId.equals(expectedId) ||
                                    (expectedId == null && expectedEvent.getType() == CREATE))) {
                        failures++;
                        printSomethingLoad();
                        System.err.println("Expected message:");
                        System.err.println(expectedId);
                        System.err.println(expectedEvent.toJson());
                        System.err.println("Received message:");
                        System.err.println(recordId);
                        System.err.println(recordEvent.toJson());
                    } else {
                        log.debug("Received message ok.");

View Full Code Here

            throw new RuntimeException(e);
        } catch (RepositoryException e) {
            throw new RuntimeException(e);
        }


        RecordEvent recordEvent;
        try {
            recordEvent = event.getRecordEvent();
        } catch (IOException e) {
            log.error("Error reading record event, processing of message cancelled", e);
            return;

View Full Code Here

0 1 2

TOP

Related Classes of org.lilyproject.util.repo.RecordEvent

org.codehaus.jackson.JsonGenerator

org.codehaus.jackson.JsonParser

org.codehaus.jackson.JsonToken

org.codehaus.jackson.util.ByteArrayBuilder

org.lilyproject.indexer.engine.test.IndexerTest

org.lilyproject.indexer.engine.test.IndexerTest$MessageVerifier

org.lilyproject.indexer.event.IndexerEditFilter

org.lilyproject.indexer.event.IndexerEditFilterTest

org.lilyproject.indexer.hbase.mapper.LilyResultToSolrMapper

org.lilyproject.indexer.hbase.mapper.LilyResultToSolrMapperTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.