Created attachment 2079974 [details]
iscsi tcmu service logs

Created attachment 2079975 [details]
iscsi service logs

Hi, I tested that issue.

I had deployed cluster with 8.0z2 GA build (the image: cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-79) and configured iscsi.

the outputs:

[root@cephnvme-vm9 ~]# docker ps
CONTAINER ID   IMAGE                                         COMMAND                  CREATED          STATUS          PORTS     NAMES
ee130c1f7fb7   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9        "/usr/bin/rbd-target…"   5 minutes ago    Up 5 minutes              ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-iscsi-iscsi-cephnvme-vm9-ncdzwz
cabf5bba0728   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9        "/usr/bin/tcmu-runner"   5 minutes ago    Up 5 minutes              ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-iscsi-iscsi-cephnvme-vm9-ncdzwz-tcmu                                                                      
25c5265a883f   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9        "/usr/bin/ceph-osd -…"   9 minutes ago    Up 9 minutes              ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-osd-2
071c8650aa8b   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9        "/usr/bin/ceph-crash…"   14 minutes ago   Up 14 minutes             ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-crash-cephnvme-vm9
c558908139e7   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9        "/usr/bin/ceph-expor…"   14 minutes ago   Up 14 minutes             ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-ceph-exporter-cephnvme-vm9
9b8690e329f7   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-79   "/usr/bin/ceph-mgr -…"   15 minutes ago   Up 15 minutes             ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-mgr-cephnvme-vm9-dguovc
0531497bbb48   cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-79   "/usr/bin/ceph-mon -…"   15 minutes ago   Up 15 minutes             ceph-c8b7baa8-0241-11f0-a1c9-02001d9b9601-mon-cephnvme-vm9

[root@cephnvme-vm9 ~]# ceph orch ls
NAME                       PORTS        RUNNING  REFRESHED  AGE  PLACEMENT
alertmanager               ?:9093,9094      0/1  -          33m  count:1
ceph-exporter                               4/4  9m ago     33m  *
crash                                       4/4  9m ago     33m  *
grafana                    ?:3000           0/1  -          33m  count:1
iscsi.iscsi                ?:5000           4/4  9m ago     23m  cephnvme-vm9;cephnvme-vm8;cephnvme-vm7;cephnvme-vm6
mgr                                         4/4  9m ago     29m  label:mgr
mon                                         4/4  9m ago     29m  label:mon
node-exporter              ?:9100           0/4  9m ago     33m  *
osd.all-available-devices                     4  9m ago     28m  *
prometheus                 ?:9095           0/1  -          33m  count:1


[root@cephnvme-vm9 ~]# ceph --version
ceph version 19.2.1 (58a7fab8be0a062d730ad7da874972fd3fba59fb) squid (stable)

[root@cephnvme-vm9 ~]# ceph versions
{
    "mon": {
        "ceph version 19.2.0-92.el9cp (6e1ac6f3b4e0616542ffeaeac53a3dff6a014e4a) squid (stable)": 4
    },
    "mgr": {
        "ceph version 19.2.0-92.el9cp (6e1ac6f3b4e0616542ffeaeac53a3dff6a014e4a) squid (stable)": 4
    },
    "osd": {
        "ceph version 19.2.0-92.el9cp (6e1ac6f3b4e0616542ffeaeac53a3dff6a014e4a) squid (stable)": 4
    },
    "tcmu-runner": {
        "ceph version 19.2.0-92.el9cp (6e1ac6f3b4e0616542ffeaeac53a3dff6a014e4a) squid (stable)": 4
    },
    "overall": {
        "ceph version 19.2.0-92.el9cp (6e1ac6f3b4e0616542ffeaeac53a3dff6a014e4a) squid (stable)": 16
    }
}


then, I upgraded ceph to image: cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-90

[root@cephnvme-vm9 ~]# ceph orch upgrade start --image cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-90

After the upgrading-

[root@cephnvme-vm9 ~]# ceph --version
ceph version 19.2.1 (58a7fab8be0a062d730ad7da874972fd3fba59fb) squid (stable)

[root@cephnvme-vm9 ~]# ceph versions
{
    "mon": {
        "ceph version 19.2.0-108.el9cp (1762f710a9f63e0304d69ed81ad964841146c93d) squid (stable)": 4
    },
    "mgr": {
        "ceph version 19.2.0-108.el9cp (1762f710a9f63e0304d69ed81ad964841146c93d) squid (stable)": 4
    },
    "osd": {
        "ceph version 19.2.0-108.el9cp (1762f710a9f63e0304d69ed81ad964841146c93d) squid (stable)": 4
    },
    "tcmu-runner": {
        "ceph version 19.2.0-108.el9cp (1762f710a9f63e0304d69ed81ad964841146c93d) squid (stable)": 4
    },
    "overall": {
        "ceph version 19.2.0-108.el9cp (1762f710a9f63e0304d69ed81ad964841146c93d) squid (stable)": 16
    }
}

[root@cephnvme-vm9 ~]# ceph orch ls
NAME                       PORTS        RUNNING  REFRESHED  AGE   PLACEMENT
alertmanager               ?:9093,9094      0/1  -          101m  count:1
ceph-exporter                               4/4  2m ago     101m  *
crash                                       4/4  2m ago     101m  *
grafana                    ?:3000           0/1  -          101m  count:1
iscsi.iscsi                ?:5000           4/4  2m ago     92m   cephnvme-vm9;cephnvme-vm8;cephnvme-vm7;cephnvme-vm6
mgr                                         4/4  2m ago     98m   label:mgr
mon                                         4/4  2m ago     98m   label:mon
node-exporter              ?:9100           0/4  2m ago     101m  *
osd.all-available-devices                     4  2m ago     97m   *
prometheus                 ?:9095           0/1  -          101m  count:1


I was not able to reproduce the issue.
note- when 8.0z2 was installed, I got ceph version 19.2.0-108.el9cp.
what was your image you built ceph?

Have deployed CEPH 8.0z2  GA kit

[ceph: root@ceph-80z3nvme-07q3fh-node1-installer /]# ceph --version
ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)

[ceph: root@ceph-80z3nvme-07q3fh-node1-installer ~]# ceph orch ls
NAME                                 PORTS             RUNNING  REFRESHED  AGE   PLACEMENT
alertmanager                         ?:9093,9094           1/1  5m ago     2h    count:1
ceph-exporter                                              6/6  10m ago    2h    *
crash                                                      6/6  10m ago    2h    *
grafana                              ?:3000                1/1  5m ago     2h    count:1
iscsi.iscsi                          ?:5000                2/2  2m ago     43m   ceph-80z3nvme-07q3fh-node4;ceph-80z3nvme-07q3fh-node5
mgr                                                        2/2  9m ago     2h    label:mgr
mon                                                        3/3  9m ago     2h    label:mon
node-exporter                        ?:9100                6/6  10m ago    2h    *
nvmeof.nvmeof_pool01.nvmeof_group01  ?:4420,5500,8009      2/2  9m ago     115m  label:nvmeof_label
osd.all-available-devices                                   16  10m ago    2h    *
prometheus                           ?:9095                1/1  5m ago     2h    count:1

Upgraded to cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-90

[ceph: root@ceph-80z3nvme-07q3fh-node1-installer /]# ceph orch upgrade start  cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-90
Initiating upgrade to cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-90
[ceph: root@ceph-80z3nvme-07q3fh-node1-installer /]# ceph orch upgrade status
{
    "target_image": "cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9@sha256:693be8216de8508dffad0452f6133c855c0700009feaa7838443557e8d0a0498",
    "in_progress": true,
    "which": "Upgrading all daemon types on all hosts",
    "services_complete": [],
    "progress": "0/46 daemons upgraded",
    "message": "Pulling cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9@sha256:693be8216de8508dffad0452f6133c855c0700009feaa7838443557e8d0a0498 image on host ceph-80z3nvme-07q3fh-node2",
    "is_paused": false
}

After upgrade

[ceph: root@ceph-80z3nvme-07q3fh-node1-installer /]# ceph -v
ceph version 19.2.0-108.el9cp (1762f710a9f63e0304d69ed81ad964841146c93d) squid (stable)

Can use cluster Installer node IP 10.0.66.93(root/passwd) for debugging

I see your ceph-common version is 19.2.0.98.

[ceph: root@ceph-80z3nvme-07q3fh-node1-installer /]# ceph --version
ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)


It appears that the specific version ceph-common-19.2.0-98.el9cp.x86_64.rpm is not available in the standard Ceph repositories. 
The closest available version is ceph-common-19.2.0-0.el9.x86_64.rpm

how did you install that version?

I have deployed 8.0z2 released version and ceph repo is https://public.dhe.ibm.com/ibmdl/export/pub/storage/ceph/8/

can find from below email 8.0z2 equivalent version is ceph-19.2.0-98.el9cp.

After deploying cluster with 8.0z2 build I have upgraded to 8.0z3 build


Greetings,

We have released IBM Ceph 8.0z2. This includes ceph-19.2.0-98.el9cp, matching the latest RH Ceph Storage 8.0z2.

You can find the RHEL repos here:

https://public.dhe.ibm.com/ibmdl/export/pub/storage/ceph/8

Containers in IBM's Production registry:

  cp.icr.io/cp/ibm-ceph/ceph-8-crimson-rhel9:19.2.0-98.0.crimson.26.20250228032923
  cp.icr.io/cp/ibm-ceph/ceph-8-rhel9:8-80
  cp.icr.io/cp/ibm-ceph/cephcsi:v4.16.8-1
  cp.icr.io/cp/ibm-ceph/grafana-rhel9:10.4.8-27
  cp.icr.io/cp/ibm-ceph/haproxy-rhel9:2.4.22-45
  cp.icr.io/cp/ibm-ceph/keepalived-rhel9:2.2.8-43
  cp.icr.io/cp/ibm-ceph/logging-loki-rhel8:v3.1.0-15
  cp.icr.io/cp/ibm-ceph/nginx-124-rhel9:1-25.1726696143
  cp.icr.io/cp/ibm-ceph/nvmeof-cli-rhel9:1.3.5-1
  cp.icr.io/cp/ibm-ceph/nvmeof-rhel9:1.3.5-3
  cp.icr.io/cp/ibm-ceph/oauth2-proxy-rhel9:v7.6.0-10
  cp.icr.io/cp/ibm-ceph/prometheus-alertmanager:v4.15.0-202502172235.p0.g870ade5.assembly.stream.el8
  cp.icr.io/cp/ibm-ceph/prometheus-node-exporter:v4.15.0-202502171304.p0.gaed837c.assembly.stream.el8
  cp.icr.io/cp/ibm-ceph/prometheus:v4.15.0-202502171304.p0.g1b43998.assembly.stream.el8
  cp.icr.io/cp/ibm-ceph/promtail-rhel9:v3.0.0-16
  cp.icr.io/cp/ibm-ceph/samba-metrics-rhel9:v0.5-1093.20250228150555.693bfad
  cp.icr.io/cp/ibm-ceph/samba-server-rhel9:v0.5-1092.20250228144921.412f1ad
  cp.icr.io/cp/ibm-ceph/snmp-notifier-rhel9:1.2.1-94

Reminder on how to get the cp user token for cp.icr.io:
https://github.ibm.com/alchemy-registry/image-iam/blob/master/obtaining_entitlement.md#obtaining-a-production-entitlement-key

I am able to reproduce this issue again another cluster when I upgrade from 8.0z2 to 8.0z3

[ceph: root@ceph-80z2to80z3-1wef1h-node1-installer /]# ceph versions
{
    "mon": {
        "ceph version 19.2.0-114.el9cp (ded81e05f6ece06e39d5564535edd25ec76f7198) squid (stable)": 3
    },
    "mgr": {
        "ceph version 19.2.0-114.el9cp (ded81e05f6ece06e39d5564535edd25ec76f7198) squid (stable)": 2
    },
    "osd": {
        "ceph version 19.2.0-114.el9cp (ded81e05f6ece06e39d5564535edd25ec76f7198) squid (stable)": 16
    },
    "tcmu-runner": {
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 2
    },
    "overall": {
        "ceph version 19.2.0-114.el9cp (ded81e05f6ece06e39d5564535edd25ec76f7198) squid (stable)": 21,
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 2
    }
}
[ceph: root@ceph-80z2to80z3-1wef1h-node1-installer /]#

When I tried upgrade from 7.1 to 8.0z3, we are not observing issue

Please use cluster 10.0.67.12(root/passwd) for debugging purpose

Hi JAYA,
1. can you share with me the log files "/var/log/ceph/<some-fsid>/tcmu-runner.log" and "/var/log/ceph/cephadm.log" please.
2. what cephadm version did you use?
3. I cannot connect to your installer. do I need any key?

Created attachment 2081193 [details]
tcmu-runner log

Created attachment 2081194 [details]
tcmu-runner log

Created attachment 2081195 [details]
cephadm log

Hi,

I have attached tcmu-runner.log and cephadm.log

I have used ceph version 19.2.0-98.el9cp to load 8.0z2 build

I have used ceph version 19.2.0-114.el9cp to load 8.0z3 build

I am able to do ssh without any key

pjayaprakash@Ps-MacBook-Pro ~ % ssh root.67.12
root.67.12's password:
Activate the web console with: systemctl enable --now cockpit.socket

Register this system with Red Hat Insights: insights-client --register
Create an account or view all your systems at https://red.ht/insights-dashboard
Last login: Thu Mar 20 08:09:38 2025 from 10.74.16.60
[root@ceph-80z2to80z3-1wef1h-node1-installer ~]# cephadm shell
Inferring fsid 46b7f330-0550-11f0-bee0-fa163e8669db
Inferring config /var/lib/ceph/46b7f330-0550-11f0-bee0-fa163e8669db/mon.ceph-80z2to80z3-1wef1h-node1-installer/config
Using ceph image with id '5673adc7ccad' and tag '8-98' created on 2025-03-19 00:13:12 +0000 UTC
cp.stg.icr.io/cp/ibm-ceph/ceph-8-rhel9@sha256:42abc709243493cfb702d26ed141f8e183982ee850c61226655223174d3b1cc6
[ceph: root@ceph-80z2to80z3-1wef1h-node1-installer /]#

verified issue and now tcmu-runner version is upgraded


CEPH versions before upgrade
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer ceph]# ceph versions
{
    "mon": {
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 3
    },
    "mgr": {
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 2
    },
    "osd": {
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 16
    },
    "tcmu-runner": {
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 2
    },
    "overall": {
        "ceph version 19.2.0-98.el9cp (d5c3cf625491b0bd76b4585e77aa0d907446f314) squid (stable)": 23
    }
}

CEPH versions after upgrade
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer /]# ceph versions
{
    "mon": {
        "ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)": 3
    },
    "mgr": {
        "ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)": 2
    },
    "osd": {
        "ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)": 16
    },
    "tcmu-runner": {
        "ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)": 2
    },
    "overall": {
        "ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)": 23
    }
}
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer /]#
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer /]# ceph --version
ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer /]# ceph -v
ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer /]# ceph version
ceph version 19.2.0-120.el9cp (7c75c2332f91a9524ad9b3bcae413f8fac4dc02c) squid (stable)
[ceph: root@ceph-80z2toz3-krmzg3-node1-installer /]#

Documentation update not required

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Important: Red Hat Ceph Storage 8.0 security, bug fix, and enhancement updates), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2025:3635