Created attachment 666688 [details] logs Description of problem: we fail to run compensation on startup for RemoveStoragePoolCommand on update of 3 tables. This caused me issues when I wanted to upgrade my setup from si25.1 to si25.2 Version-Release number of selected component (if applicable): si25.1 How reproducible: Steps to Reproduce: 1. remove storage pool 2. restart engine 3. Actual results: engine will fail to run compensation on 3 tables for RemoveStoragePoolCommand Expected results: we should be able to run compensation on RemoveStoragePoolCommand Additional info: logs 2012-12-20 15:26:55,383 INFO [org.ovirt.engine.core.utils.transaction.TransactionSupport] (ServerService Thread Pool -- 34) transaction rolled back 2012-12-20 15:26:55,390 ERROR [org.ovirt.engine.core.bll.Backend] (ServerService Thread Pool -- 34) Failed to run compensation on startup for Command org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand, Command Id : 781741e4-ed9a -4827-b5da-237166cee0d0, due to: DataIntegrityViolationException: CallableStatementCallback; SQL [{call insertstorage_pool_iso_map(?, ?, ?, ?)}]; ERROR: insert or update on table "storage_pool_iso_map" violates foreign key constraint "f k_storage_domain_pool_map_storage_pool" Detail: Key (storage_pool_id)=(e1180de1-7b00-4b77-8cf2-102112a3ca38) is not present in table "storage_pool". Where: SQL statement "INSERT INTO storage_pool_iso_map(storage_id, storage_pool_id, status, owner) VALUES( $1 , $2 , $3 , $4 )" PL/pgSQL function "insertstorage_pool_iso_map" line 2 at SQL statement; nested exception is org.postgresql.util.PSQLException: ERROR: insert or update on table "storage_pool_iso_map" violates foreign key constraint "fk_storage_domain_po ol_map_storage_pool" Detail: Key (storage_pool_id)=(e1180de1-7b00-4b77-8cf2-102112a3ca38) is not present in table "storage_pool". Where: SQL statement "INSERT INTO storage_pool_iso_map(storage_id, storage_pool_id, status, owner) VALUES( $1 , $2 , $3 , $4 )" PL/pgSQL function "insertstorage_pool_iso_map" line 2 at SQL statement: org.springframework.dao.DataIntegrityViolationException: CallableStatementCallback; SQL [{call insertstorage_pool_iso_map(?, ?, ?, ?)}]; ERROR: insert or update on table "storage_pool_iso_map" violates foreign key constraint "fk_storage_domain_pool_map_storage_pool" Detail: Key (storage_pool_id)=(e1180de1-7b00-4b77-8cf2-102112a3ca38) is not present in table "storage_pool". Where: SQL statement "INSERT INTO storage_pool_iso_map(storage_id, storage_pool_id, status, owner) VALUES( $1 , $2 , $3 , $4 )" PL/pgSQL function "insertstorage_pool_iso_map" line 2 at SQL statement; nested exception is org.postgresql.util.PSQLException: ERROR: insert or update on table "storage_pool_iso_map" violates foreign key constraint "fk_storage_domain_po ol_map_storage_pool" Detail: Key (storage_pool_id)=(e1180de1-7b00-4b77-8cf2-102112a3ca38) is not present in table "storage_pool". Where: SQL statement "INSERT INTO storage_pool_iso_map(storage_id, storage_pool_id, status, owner) VALUES( $1 , $2 , $3 , $4 )" PL/pgSQL function "insertstorage_pool_iso_map" line 2 at SQL statement at org.springframework.jdbc.support.SQLErrorCodeSQLExceptionTranslator.doTranslate(SQLErrorCodeSQLExceptionTranslator.java:245) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:72) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:1030) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.springframework.jdbc.core.JdbcTemplate.call(JdbcTemplate.java:1064) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.springframework.jdbc.core.simple.AbstractJdbcCall.executeCallInternal(AbstractJdbcCall.java:388) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.springframework.jdbc.core.simple.AbstractJdbcCall.doExecute(AbstractJdbcCall.java:351) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.springframework.jdbc.core.simple.SimpleJdbcCall.execute(SimpleJdbcCall.java:181) [spring-jdbc-3.1.1.RELEASE.jar:3.1.1.RELEASE] at org.ovirt.engine.core.dal.dbbroker.SimpleJdbcCallsHandler.executeImpl(SimpleJdbcCallsHandler.java:124) [engine-dal.jar:] at org.ovirt.engine.core.dal.dbbroker.SimpleJdbcCallsHandler.executeModification(SimpleJdbcCallsHandler.java:37) [engine-dal.jar:] at org.ovirt.engine.core.dao.StoragePoolIsoMapDAODbFacadeImpl.save(StoragePoolIsoMapDAODbFacadeImpl.java:44) [engine-dal.jar:] at org.ovirt.engine.core.dao.StoragePoolIsoMapDAODbFacadeImpl.save(StoragePoolIsoMapDAODbFacadeImpl.java:15) [engine-dal.jar:] at org.ovirt.engine.core.bll.CommandBase$1.runInTransaction(CommandBase.java:374) [engine-bll.jar:] at org.ovirt.engine.core.utils.transaction.TransactionSupport.executeInNewTransaction(TransactionSupport.java:204) [engine-utils.jar:] at org.ovirt.engine.core.bll.CommandBase.internalCompensate(CommandBase.java:341) [engine-bll.jar:] at org.ovirt.engine.core.bll.CommandBase.compensate(CommandBase.java:328) [engine-bll.jar:] at org.ovirt.engine.core.bll.Backend.compensate(Backend.java:272) [engine-bll.jar:] at org.ovirt.engine.core.bll.Backend.Initialize(Backend.java:168) [engine-bll.jar:] at org.ovirt.engine.core.bll.Backend.create(Backend.java:119) [engine-bll.jar:] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [rt.jar:1.7.0_09-icedtea] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) [rt.jar:1.7.0_09-icedtea] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.7.0_09-icedtea] at java.lang.reflect.Method.invoke(Method.java:601) [rt.jar:1.7.0_09-icedtea]
[root@dafna-web-admin tmp]# psql --expanded -U postgres engine -c "SELECT * from business_entity_snapshot" Password for user postgres: psql: FATAL: password authentication failed for user "postgres" [root@dafna-web-admin tmp]# psql --expanded -U postgres engine -c "SELECT * from business_entity_snapshot" Password for user postgres: -[ RECORD 1 ]---+------------------------------------------------------------------------------------------------------- id | 11204846-4468-11e2-a9a7-001a4a169741 command_id | 781741e4-ed9a-4827-b5da-237166cee0d0 command_type | org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand entity_id | 2b3dabcf-ed47-4bb6-b9c9-ef580e763728 entity_type | org.ovirt.engine.core.common.businessentities.Network entity_snapshot | { : "@class" : "org.ovirt.engine.core.common.businessentities.Network", : "id" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "2b3dabcf-ed47-4bb6-b9c9-ef580e763728" : } ], : "description" : "Management Network", : "type" : null, : "addr" : null, : "subnet" : null, : "gateway" : null, : "vlan_id" : null, : "stp" : false, : "storage_pool_id" : [ "org.ovirt.engine.core.compat.NGuid", { : "uuid" : "e1180de1-7b00-4b77-8cf2-102112a3ca38" : } ], : "cluster" : null, : "vmNetwork" : true, : "mtu" : 0 : } snapshot_class | org.ovirt.engine.core.common.businessentities.Network snapshot_type | 0 insertion_order | 1 started_at | 2012-12-12 16:27:32.903347+02 -[ RECORD 2 ]---+------------------------------------------------------------------------------------------------------- id | 1129745c-4468-11e2-a90b-001a4a169741 command_id | 781741e4-ed9a-4827-b5da-237166cee0d0 command_type | org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand entity_id | storagePoolId = e1180de1-7b00-4b77-8cf2-102112a3ca38, storageId = 484eb3e8-abfb-467b-9d3d-4aa621873262 entity_type | org.ovirt.engine.core.common.businessentities.storage_pool_iso_map entity_snapshot | { : "id" : { : "storageId" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "484eb3e8-abfb-467b-9d3d-4aa621873262" : } ], : "storagePoolId" : [ "org.ovirt.engine.core.compat.NGuid", { : "uuid" : "e1180de1-7b00-4b77-8cf2-102112a3ca38" : } ] : }, : "status" : "InActive", : "storage_pool_id" : [ "org.ovirt.engine.core.compat.NGuid", { : "uuid" : "e1180de1-7b00-4b77-8cf2-102112a3ca38" : } ], : "storage_id" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "484eb3e8-abfb-467b-9d3d-4aa621873262" : } ], : "owner" : "Unknown" : } snapshot_class | org.ovirt.engine.core.common.businessentities.storage_pool_iso_map snapshot_type | 0 insertion_order | 2 started_at | 2012-12-12 16:27:32.974385+02 -[ RECORD 3 ]---+------------------------------------------------------------------------------------------------------- id | 113d1458-4468-11e2-bf45-001a4a169741 command_id | 781741e4-ed9a-4827-b5da-237166cee0d0 command_type | org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand entity_id | 484eb3e8-abfb-467b-9d3d-4aa621873262 entity_type | org.ovirt.engine.core.common.businessentities.storage_domain_static entity_snapshot | { : "id" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "484eb3e8-abfb-467b-9d3d-4aa621873262" : } ], : "storage" : "95416488-9e8f-4b3c-8f52-0a69144bcfd9", : "connection" : null, : "storageFormat" : "V3", : "autoRecoverable" : true, : "storage_pool_name" : null, : "storage_domain_type" : "Master", : "storage_type" : "POSIXFS", : "storage_name" : "ext4", : "lastTimeUsedAsMaster" : 0 : } snapshot_class | org.ovirt.engine.core.common.businessentities.storage_domain_static snapshot_type | 0 insertion_order | 3 started_at | 2012-12-12 16:27:33.10247+02 [root@dafna-web-admin tmp]#
Fixed at commit - 623303914af756e55a95b86f68787a4d603729cc Change-Id: I09d8322c87bd4981c232fa8217532a541e24cc63
verified on sf5 [root@dafna-web-admin ~]# psql --expanded -U postgres engine -c "SELECT * from business_entity_snapshot" Password for user postgres: -[ RECORD 1 ]---+------------------------------------------------------------------------------------------------------- id | ac2a7088-da5d-4f3f-b3d8-99ea32ccc881 command_id | d5076978-7a1c-4caa-b43a-84ca6580c0f0 command_type | org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand entity_id | d1d3bf5c-677c-4421-ab31-2e856577ee68 entity_type | org.ovirt.engine.core.common.businessentities.network.Network entity_snapshot | { : "@class" : "org.ovirt.engine.core.common.businessentities.network.Network", : "id" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "d1d3bf5c-677c-4421-ab31-2e856577ee68" : } ], : "name" : "rhevm", : "description" : "Management Network", : "type" : null, : "addr" : null, : "subnet" : null, : "gateway" : null, : "vlanId" : null, : "stp" : false, : "dataCenterId" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "cbaa914a-a04e-40bf-bc41-3c1cdfefe5c1" : } ], : "cluster" : null, : "vmNetwork" : true, : "mtu" : 0 : } snapshot_class | org.ovirt.engine.core.common.businessentities.network.Network snapshot_type | 0 insertion_order | 1 started_at | 2013-02-06 17:56:38.79503+02 -[ RECORD 2 ]---+------------------------------------------------------------------------------------------------------- id | 9c641eab-5898-4526-a5fa-0a9ea365edd1 command_id | d5076978-7a1c-4caa-b43a-84ca6580c0f0 command_type | org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand entity_id | storagePoolId = cbaa914a-a04e-40bf-bc41-3c1cdfefe5c1, storageId = 67e91f21-4045-4179-901c-a5e7911a1b70 entity_type | org.ovirt.engine.core.common.businessentities.StoragePoolIsoMap entity_snapshot | { : "id" : { : "storageId" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "67e91f21-4045-4179-901c-a5e7911a1b70" : } ], : "storagePoolId" : [ "org.ovirt.engine.core.compat.NGuid", { : "uuid" : "cbaa914a-a04e-40bf-bc41-3c1cdfefe5c1" : } ] : }, : "storage_id" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "67e91f21-4045-4179-901c-a5e7911a1b70" : } ], : "storage_pool_id" : [ "org.ovirt.engine.core.compat.NGuid", { : "uuid" : "cbaa914a-a04e-40bf-bc41-3c1cdfefe5c1" : } ], : "owner" : "Unknown", : "status" : "Maintenance" : } snapshot_class | org.ovirt.engine.core.common.businessentities.StoragePoolIsoMap snapshot_type | 0 insertion_order | 2 started_at | 2013-02-06 17:56:38.899412+02 -[ RECORD 3 ]---+------------------------------------------------------------------------------------------------------- id | ad1f70d4-ddf9-46c3-8129-0b8ad2518fe2 command_id | d5076978-7a1c-4caa-b43a-84ca6580c0f0 command_type | org.ovirt.engine.core.bll.storage.RemoveStoragePoolCommand entity_id | 67e91f21-4045-4179-901c-a5e7911a1b70 entity_type | org.ovirt.engine.core.common.businessentities.StorageDomainStatic entity_snapshot | { : "id" : [ "org.ovirt.engine.core.compat.Guid", { : "uuid" : "67e91f21-4045-4179-901c-a5e7911a1b70" : } ], : "storage" : "J6OQF6-yRMB-jrJO-qJ4U-uFyl-SFBh-u4PMKb", : "connection" : null, : "storageFormat" : "V2", : "autoRecoverable" : true, : "storage_type" : "ISCSI", : "storage_name" : "Kill", : "lastTimeUsedAsMaster" : 0, : "storage_domain_type" : "Master", : "storage_pool_name" : null : } snapshot_class | org.ovirt.engine.core.common.businessentities.StorageDomainStatic snapshot_type | 0 insertion_order | 3 started_at | 2013-02-06 17:56:44.25269+02 [root@dafna-web-admin ~]# !less less /var/log/ovirt-engine/engine.log [root@dafna-web-admin ~]# psql --expanded -U postgres engine -c "SELECT * from business_entity_snapshot" Password for user postgres: psql: FATAL: password authentication failed for user "postgres" [root@dafna-web-admin ~]# psql --expanded -U postgres engine -c "SELECT * from business_entity_snapshot" Password for user postgres: (No rows) [root@dafna-web-admin ~]#
3.2 has been released