Note: This bug is displayed in read-only format because the product is no longer active in Red Hat Bugzilla.

Bug 1990446

Summary: Concurrent deploy of 2 host fails during certificate creation
Product: [oVirt] ovirt-engine Reporter: Martin Perina <mperina>
Component: ovirt-host-deploy-ansibleAssignee: Dana <delfassy>
Status: CLOSED CURRENTRELEASE QA Contact: Pavol Brilla <pbrilla>
Severity: high Docs Contact:
Priority: high    
Version: 4.4.8CC: bugs
Target Milestone: ovirt-4.5.0Flags: mperina: ovirt-4.5+
Target Release: 4.5.0   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: ovirt-engine-4.5.0 Doc Type: No Doc Update
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2022-05-23 06:21:25 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: Infra RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:

Description Martin Perina 2021-08-05 12:25:52 UTC
When 2 host deploy processes started at the exact same time, one of them will fail during certificate creation with below error:


2021-08-05 10:48:52 CEST - TASK [ovirt-host-deploy-vdsm-certificates : Run PKI enroll request for vdsm and QEMU] ***
2021-08-05 10:48:55 CEST - failed: [ost-basic-suite-master-host-0] (item={'ou': '', 'ca_file': 'ca', 'cert_dir': 'certs', 'req_dir': 'requests'}) => {"ansible_loop_var": "item", "changed": true, "cmd": ["/usr/share/ovirt-engine/bin/pki-enroll-request.sh", "--name=ost-basic-suite-master-host-0", "--subject=/O=Test/CN=ost-basic-suite-master-host-0", "--san=DNS:ost-basic-suite-master-host-0", "--days=398", "--timeout=30", "--ca-file=ca", "--cert-dir=certs", "--req-dir=requests"], "delta": "0:00:02.045767", "end": "2021-08-05 10:48:54.123758", "item": {"ca_file": "ca", "cert_dir": "certs", "ou": "", "req_dir": "requests"}, "msg": "non-zero return code", "rc": 1, "start": "2021-08-05 10:48:52.077991", "stderr": "Cannot remove '/tmp/ovirt-engine-pki.v2.lock' please remove manually", "stderr_lines": ["Cannot remove '/tmp/ovirt-engine-pki.v2.lock' please remove manually"], "stdout": "", "stdout_lines": []}
2021-08-05 10:48:55 CEST - changed: [ost-basic-suite-master-host-0] => (item={'ou': '/OU=qemu', 'ca_file': 'qemu-ca', 'cert_dir': 'certs-qemu', 'req_dir': 'requests-qemu'})
2021-08-05 10:48:55 CEST - 
2021-08-05 10:48:55 CEST - {
  "status" : "OK",
  "msg" : "",
  "data" : {
    "uuid" : "96ffb988-fdd5-47f2-8394-66461d9d624c",
    "counter" : 139,
    "stdout" : "",
    "start_line" : 135,
    "end_line" : 135,
    "runner_ident" : "e8a030da-f5c9-11eb-97d6-5452c0a8d102",
    "event" : "runner_on_failed",
    "pid" : 49432,
    "created" : "2021-08-05T08:48:54.443621",
    "parent_uuid" : "5452c0a8-d102-7825-869d-000000000238",
    "event_data" : {
      "playbook" : "ovirt-host-deploy.yml",
      "playbook_uuid" : "09d1e410-3bd6-4530-8134-0fa72374b74f",
      "play" : "all",
      "play_uuid" : "5452c0a8-d102-7825-869d-000000000006",
      "play_pattern" : "all",
      "task" : "Run PKI enroll request for vdsm and QEMU",
      "task_uuid" : "5452c0a8-d102-7825-869d-000000000238",
      "task_action" : "command",
      "task_args" : "",
      "task_path" : "/usr/share/ovirt-engine/ansible-runner-service-project/project/roles/ovirt-host-deploy-vdsm-certificates/tasks/main.yml:38",
      "role" : "ovirt-host-deploy-vdsm-certificates",
      "host" : "ost-basic-suite-master-host-0",
      "remote_addr" : "ost-basic-suite-master-host-0",
      "res" : {
        "results" : [ {
          "msg" : "non-zero return code",
          "cmd" : [ "/usr/share/ovirt-engine/bin/pki-enroll-request.sh", "--name=ost-basic-suite-master-host-0", "--subject=/O=Test/CN=ost-basic-suite-master-host-0", "--san=DNS:ost-basic-suite-master-host-0", "--days=398", "--timeout=30", "--ca-file=ca", "--cert-dir=certs", "--req-dir=requests" ],
          "stdout" : "",
          "stderr" : "Cannot remove '/tmp/ovirt-engine-pki.v2.lock' please remove manually",
          "rc" : 1,
          "start" : "2021-08-05 10:48:52.077991",
          "end" : "2021-08-05 10:48:54.123758",
          "delta" : "0:00:02.045767",
          "changed" : true,
          "failed" : true,
          "invocation" : {
            "module_args" : {
              "_raw_params" : "\"/usr/share/ovirt-engine/bin/pki-enroll-request.sh\"\n\"--name=ost-basic-suite-master-host-0\"\n\"--subject=/O=Test/CN=ost-basic-suite-master-host-0\"\n\"--san=DNS:ost-basic-suite-master-host-0\"\n\"--days=398\"\n\"--timeout=30\"\n\"--ca-file=ca\"\n\"--cert-dir=certs\"\n\"--req-dir=requests\"\n",
              "warn" : true,
              "_uses_shell" : false,
              "stdin_add_newline" : true,
              "strip_empty_ends" : true,
              "argv" : null,
              "chdir" : null,
              "executable" : null,
              "creates" : null,
              "removes" : null,
              "stdin" : null
            }
          },
          "stdout_lines" : [ ],
          "stderr_lines" : [ "Cannot remove '/tmp/ovirt-engine-pki.v2.lock' please remove manually" ],
          "_ansible_no_log" : false,
          "item" : {
            "ou" : "",
            "ca_file" : "ca",
            "cert_dir" : "certs",
            "req_dir" : "requests"
          },
          "ansible_loop_var" : "item",
          "_ansible_item_label" : {
            "ou" : "",
            "ca_file" : "ca",
            "cert_dir" : "certs",
            "req_dir" : "requests"
          }
        }, {
          "cmd" : [ "/usr/share/ovirt-engine/bin/pki-enroll-request.sh", "--name=ost-basic-suite-master-host-0", "--subject=/O=Test/CN=ost-basic-suite-master-host-0/OU=qemu", "--san=DNS:ost-basic-suite-master-host-0", "--days=398", "--timeout=30", "--ca-file=qemu-ca", "--cert-dir=certs-qemu", "--req-dir=requests-qemu" ],
          "stdout" : "",
          "stderr" : "Using configuration from openssl.conf\nCheck that the request matches the signature\nSignature ok\nThe Subject's Distinguished Name is as follows\norganizationName      :ASN.1 12:'Test'\ncommonName            :ASN.1 12:'ost-basic-suite-master-host-0'\norganizationalUnitName:ASN.1 12:'qemu'\nCertificate is to be certified until Sep  7 08:48:54 2022 GMT (398 days)\n\nWrite out database with 1 new entries\nData Base Updated",
          "rc" : 0,
          "start" : "2021-08-05 10:48:54.361248",
          "end" : "2021-08-05 10:48:54.420031",
          "delta" : "0:00:00.058783",
          "changed" : true,
          "invocation" : {
            "module_args" : {
              "_raw_params" : "\"/usr/share/ovirt-engine/bin/pki-enroll-request.sh\"\n\"--name=ost-basic-suite-master-host-0\"\n\"--subject=/O=Test/CN=ost-basic-suite-master-host-0/OU=qemu\"\n\"--san=DNS:ost-basic-suite-master-host-0\"\n\"--days=398\"\n\"--timeout=30\"\n\"--ca-file=qemu-ca\"\n\"--cert-dir=certs-qemu\"\n\"--req-dir=requests-qemu\"\n",
              "warn" : true,
              "_uses_shell" : false,
              "stdin_add_newline" : true,
              "strip_empty_ends" : true,
              "argv" : null,
              "chdir" : null,
              "executable" : null,
              "creates" : null,
              "removes" : null,
              "stdin" : null
            }
          },
          "stdout_lines" : [ ],
          "stderr_lines" : [ "Using configuration from openssl.conf", "Check that the request matches the signature", "Signature ok", "The Subject's Distinguished Name is as follows", "organizationName      :ASN.1 12:'Test'", "commonName            :ASN.1 12:'ost-basic-suite-master-host-0'", "organizationalUnitName:ASN.1 12:'qemu'", "Certificate is to be certified until Sep  7 08:48:54 2022 GMT (398 days)", "", "Write out database with 1 new entries", "Data Base Updated" ],
          "_ansible_no_log" : false,
          "failed" : false,
          "item" : {
            "ou" : "/OU=qemu",
            "ca_file" : "qemu-ca",
            "cert_dir" : "certs-qemu",
            "req_dir" : "requests-qemu"
          },
          "ansible_loop_var" : "item",
          "_ansible_item_label" : {
            "ou" : "/OU=qemu",
            "ca_file" : "qemu-ca",
            "cert_dir" : "certs-qemu",
            "req_dir" : "requests-qemu"
          }
        } ],
        "changed" : true,
        "msg" : "All items completed"
      },
      "start" : "2021-08-05T08:48:51.867281",
      "end" : "2021-08-05T08:48:54.443502",
      "duration" : 2.576221,
      "ignore_errors" : null,
      "event_loop" : "items",
      "uuid" : "96ffb988-fdd5-47f2-8394-66461d9d624c"
    }
  }
}

Comment 6 Pavol Brilla 2022-05-04 11:07:44 UTC
Software Version:4.5.0.5-0.7.el8ev

Triggering enroll certificates on 2 hosts through api did not throw errors during enrolling