Bug 1949097

I'm going to close this as a duplicate

*** This bug has been marked as a duplicate of bug 1945075 ***

*** Bug 1962497 has been marked as a duplicate of this bug. ***

I just can't reproduce this. When I deploy latest 16.1 I get following rsyslog config on compute node:

[heat-admin@compute-0 ~]$ cat /var/lib/config-data/puppet-generated/rsyslog/etc/rsyslog.d/50_openstack_logs.conf
module(load="imfile")
module(load="omelasticsearch")
# nova_compute_openstack.nova.compute
input(type="imfile"
  file="/var/log/containers/nova/nova-compute.log"
  tag="openstack.nova.compute"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)

template (name="rsyslog-node-index" type="list"
)
{
    constant(value="compute-0.redhat.local-" )
    property(dateFormat="year" name="timereported" )
    constant(value="." )
    property(dateFormat="month" name="timereported" )
    constant(value="." )
    property(dateFormat="day" name="timereported" )
}

template (name="rsyslog-record" type="list"
 option.jsonf="on")
{
    property(format="jsonf" name="timereported" outname="@timestamp" )
    property(format="jsonf" name="hostname" outname="host" )
    property(format="jsonf" name="syslogseverity-text" outname="severity" )
    property(format="jsonf" name="syslogfacility-text" outname="facility" )
    property(format="jsonf" name="syslogtag" outname="tag" )
    property(format="jsonf" name="app-name" outname="source" )
    property(format="jsonf" name="msg" outname="message" )
}
# elasticsearch
action(type="omelasticsearch"
    name="elasticsearch"
    dynSearchIndex="on"
    errorfile="/var/log/rsyslog/omelasticsearch.log"
    searchIndex="rsyslog-node-index"
    template="rsyslog-record"
  )
[heat-admin@compute-0 ~]$ 
[heat-admin@compute-0 ~]$ sudo podman logs rsyslog
+ sudo -E kolla_set_configs
INFO:__main__:Loading config file at /var/lib/kolla/config_files/config.json
INFO:__main__:Validating config file
INFO:__main__:Kolla config strategy set to: COPY_ALWAYS
INFO:__main__:Copying service configuration files
INFO:__main__:Copying /var/lib/kolla/config_files/src/etc/rsyslog.d/50_openstack_logs.conf to /etc/rsyslog.d/50_openstack_logs.conf
INFO:__main__:Deleting /etc/rsyslog.conf
INFO:__main__:Copying /var/lib/kolla/config_files/src/etc/rsyslog.conf to /etc/rsyslog.conf
INFO:__main__:Writing out command to execute
INFO:__main__:Setting permission for /var/lib/rsyslog
INFO:__main__:Setting permission for /var/log/rsyslog
++ cat /run_command
+ CMD='/usr/sbin/rsyslogd -n'
+ ARGS=
+ sudo kolla_copy_cacerts
Running command: '/usr/sbin/rsyslogd -n'
+ [[ ! -n '' ]]
+ . kolla_extend_start
+ echo 'Running command: '\''/usr/sbin/rsyslogd -n'\'''
+ exec /usr/sbin/rsyslogd -n
rsyslogd: imfile: no working or state file directory set, imfile will create state files in the current working directory (probably the root dir). Use global(workDirectory="/some/path") to set the working directory [v8.1911.0-3.el8 try https://www.rsyslog.com/e/2450 ]
rsyslogd: imfile: on startup file '/var/log/containers/nova/nova-compute.log' does not exist but is configured in static file monitor - this may indicate a misconfiguration. If the file appears at a later time, it will automatically be processed. Reason: No such file or directory [v8.1911.0-3.el8]
rsyslogd: omelasticsearch: we are suspending ourselfs due to server failure 7: Failed to connect to localhost port 9200: Connection refused [v8.1911.0-3.el8 try https://www.rsyslog.com/e/2007 ]
[heat-admin@compute-0 ~]$

We can ignore the error on nova-compute.log since rsyslog is being deployed before nova, so the log did not exist when rsyslog started, but was created a while after.

[heat-admin@compute-0 ~]$ sudo ls -l /var/log/containers/nova/
total 7936
-rw-r--r--. 1 42436 42436 4041700 Jul  1 20:23 nova-compute.log
-rw-r--r--. 1 42436 42436       0 Jul  1 14:44 nova-manage.log
[heat-admin@compute-0 ~]$ 
[heat-admin@compute-0 ~]$ sudo podman exec -it rsyslog ls -l /var/log/containers/nova/
total 7936
-rw-r--r--. 1 nova nova 4055775 Jul  1 20:24 nova-compute.log
-rw-r--r--. 1 nova nova       0 Jul  1 14:44 nova-manage.log
[heat-admin@compute-0 ~]$ 

And yes I used instanceha for deployment:
(.venv-infrared) [root@lenovo-p720-rdo-13 infrared]$ grep instanceha osp16-user-config.yaml -B10 -A2
rhos_release_rpm: "http://rhos-release.virt.bos.redhat.com/repos/rhos-release/rhos-release-latest.noarch.rpm"

tripleo_heat_templates: #[]
  # STF deploy
  #- /usr/share/openstack-tripleo-heat-templates/environments/metrics/qdr-edge-only.yaml
  #- /usr/share/openstack-tripleo-heat-templates/environments/metrics/collectd-write-qdr.yaml
  #- /usr/share/openstack-tripleo-heat-templates/environments/metrics/ceilometer-write-qdr.yaml
  # logging deploy
  - /usr/share/openstack-tripleo-heat-templates/environments/logging-environment-rsyslog.yaml
  # BZ1949097
  - /usr/share/openstack-tripleo-heat-templates/environments/compute-instanceha.yaml

custom_templates:
(.venv-infrared) [root@lenovo-p720-rdo-13 infrared]$ 
(.venv-infrared) [root@lenovo-p720-rdo-13 infrared]$ grep osp16-user-config.yaml deploy_osp16.sh -B13 -A3
infrared tripleo-overcloud \
    -o overcloud-install.yml \
    --version $OSP_VERSION \
    --deployment-files virt \
    --network-backend geneve \
    --network-protocol ipv4 \
    --storage-external no \
    --overcloud-ssl no \
    --introspect yes \
    --tagging yes \
    --deploy yes \
    --containers yes \
    --ntp-server "${NTP_SERVER}" \
    --overcloud-templates osp16-user-config.yaml \
    --registry-mirror $CONTAINER_REGISTRY \
    --registry-undercloud-skip no && \
\
(.venv-infrared) [root@lenovo-p720-rdo-13 infrared]$ grep OSP_VERSION= deploy_osp16.sh
OSP_VERSION=16.1
(.venv-infrared) [root@lenovo-p720-rdo-13 infrared]$

Were you using latest OSP16.1 or you are running some older z-release? Can you provide me exact reproducer please?

Any update here?

I was not able to reproduce that on 16.2 too.

[root@compute-0 ~]# podman ps | grep rsys
b090d9c2dc62  undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhosp16-openstack-rsyslog:16.2_20210901.2                     kolla_start  23 hours ago  Up About a minute ago          rsyslog
[root@compute-0 ~]# podman logs rsyslog
+ sudo -E kolla_set_configs
+ sudo -E kolla_set_configs
INFO:__main__:Loading config file at /var/lib/kolla/config_files/config.json
INFO:__main__:Validating config file
INFO:__main__:Kolla config strategy set to: COPY_ALWAYS
INFO:__main__:Copying service configuration files
INFO:__main__:Deleting /etc/rsyslog.d/50_openstack_logs.conf
INFO:__main__:Copying /var/lib/kolla/config_files/src/etc/rsyslog.d/50_openstack_logs.conf to /etc/rsyslog.d/50_openstack_logs.conf
INFO:__main__:Deleting /etc/rsyslog.conf
INFO:__main__:Copying /var/lib/kolla/config_files/src/etc/rsyslog.conf to /etc/rsyslog.conf
INFO:__main__:Writing out command to execute
INFO:__main__:Setting permission for /var/lib/rsyslog
INFO:__main__:Setting permission for /var/log/rsyslog
++ cat /run_command
+ CMD='/usr/sbin/rsyslogd -n'
+ ARGS=
+ [[ ! -n '' ]]
+ . kolla_extend_start
+ echo 'Running command: '\''/usr/sbin/rsyslogd -n'\'''
Running command: '/usr/sbin/rsyslogd -n'
+ exec /usr/sbin/rsyslogd -n
rsyslogd: imfile: no working or state file directory set, imfile will create state files in the current working directory (probably the root dir). Use global(workDirectory="/some/path") to set the working directory [v8.1911.0-7.el8_4.2 try https://www.rsyslog.com/e/2450 ]
rsyslogd: omelasticsearch: we are suspending ourselfs due to server failure 7: Failed to connect to localhost port 9200: Connection refused [v8.1911.0-7.el8_4.2 try https://www.rsyslog.com/e/2007 ]
[root@compute-0 ~]#
[root@compute-0 ~]# podman exec -it rsyslog cat /etc/rsyslog.d/50_openstack_logs.conf
module(load="imfile")
module(load="omelasticsearch")
# nova_compute_openstack.nova.compute
input(type="imfile"
  file="/var/log/containers/nova/nova-compute.log"
  tag="openstack.nova.compute"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)

template (name="rsyslog-node-index" type="list"
)
{
    constant(value="compute-0.redhat.local-" )
    property(dateFormat="year" name="timereported" )
    constant(value="." )
    property(dateFormat="month" name="timereported" )
    constant(value="." )
    property(dateFormat="day" name="timereported" )
}

template (name="rsyslog-record" type="list"
 option.jsonf="on")
{
    property(dateFormat="rfc3339" format="jsonf" name="timereported" outname="@timestamp" )
    property(format="jsonf" name="hostname" outname="host" )
    property(format="jsonf" name="syslogseverity" outname="severity" )
    property(format="jsonf" name="syslogfacility-text" outname="facility" )
    property(format="jsonf" name="syslogtag" outname="tag" )
    property(format="jsonf" name="app-name" outname="source" )
    property(format="jsonf" name="msg" outname="message" )
    property(format="jsonf" name="$!metadata!filename" outname="file" )
    constant(format="jsonf" outname="cloud" value="overcloud.localdomain" )
    constant(format="jsonf" outname="region" value="regionOne" )
}
# elasticsearch
action(type="omelasticsearch"
    name="elasticsearch"
    dynSearchIndex="on"
    errorfile="/var/log/rsyslog/omelasticsearch.log"
    searchIndex="rsyslog-node-index"
    template="rsyslog-record"
  )
[root@compute-0 ~]#

Ok, in that case I suggest to update customers environment to latest 16.1 z-release or redeploy with NovaComputeLoggingSource parameter value from comment #17.