Bug 1777235

Patches are already in the release. We just need to test that.

Renamed the title to not feel so massive and reduced the priority/severity from high to medium.

Did some testing, and deployment definitely works, but some of the things I looked at was from the original comment:

1.non-existent log files are configured
  -- I thought I saw this once, but I can't seem to reproduce on the latest deployment
2.nova-compute.log is configured for controller
  -- seems to be resolved
3.use case for missing logging_sources configuration for composable services is missing
  -- not sure about this

My deployment looked like this:

```
parameter_defaults:
    RsyslogElasticsearchSetting:
        uid: "elastic"
        pwd: "elastic"
        server: "https://store-openstack-logging.apps.stftest1.lab"
        serverport: 443
    RsyslogElasticsearchTlsCACert: |
        -----BEGIN CERTIFICATE-----
...
        -----END CERTIFICATE-----
    RsyslogElasticsearchTlsClientCert: |
        -----BEGIN CERTIFICATE-----
...
        -----END CERTIFICATE-----
    RsyslogElasticsearchTlsClientKey: |
        -----BEGIN RSA PRIVATE KEY-----
...
        -----END RSA PRIVATE KEY-----
```

Overcloud deployment looked like:

```
#!/bin/bash

openstack overcloud deploy \
--timeout 100 \
--templates /usr/share/openstack-tripleo-heat-templates \
--stack overcloud \
--libvirt-type kvm \
-e /home/stack/virt/config_lvm.yaml \
-e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml \
-e /usr/share/openstack-tripleo-heat-templates/environments/logging-environment-rsyslog.yaml \
-e /home/stack/virt/network/network-environment.yaml \
-e /home/stack/virt/network/dvr-override.yaml \
-e /home/stack/virt/inject-trust-anchor.yaml \
-e /home/stack/virt/hostnames.yml \
-e /home/stack/virt/debug.yaml \
-e /home/stack/virt/nodes_data.yaml \
-e ~/containers-prepare-parameter.yaml \
-e /home/stack/virt/stf-connectors.yaml \
-e /home/stack/virt/logging-connector.yaml \
--log-file overcloud_deployment_43.log

```

(Note this is an STF environment but I don't expect any negative interactions between the client configuration here, and we want these to co-exist.)

Deployment looks fine, and results in this configuration on a controller-0:

## controller-0

```
# cat 50_openstack_logs.conf 
module(load="imfile")
module(load="omelasticsearch")
# ceilometer_agent_central_openstack.ceilometer.agent.central
input(type="imfile"
  file="/var/log/containers/ceilometer/central.log"
  tag="openstack.ceilometer.agent.central"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# ceilometer_agent_notification_openstack.ceilometer.agent.notification
input(type="imfile"
  file="/var/log/containers/ceilometer/agent-notification.log"
  tag="openstack.ceilometer.agent.notification"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# cinder_scheduler_openstack.cinder.scheduler
input(type="imfile"
  file="/var/log/containers/cinder/cinder-scheduler.log"
  tag="openstack.cinder.scheduler"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# cinder_volume_openstack.cinder.volume
input(type="imfile"
  file="/var/log/containers/cinder/cinder-volume.log"
  tag="openstack.cinder.volume"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# glance_api_openstack.glance.api
input(type="imfile"
  file="/var/log/containers/glance/api.log"
  tag="openstack.glance.api"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# heat_api_cfn_openstack.heat.api.cfn
input(type="imfile"
  file="/var/log/containers/heat/heat_api_cfn.log"
  tag="openstack.heat.api.cfn"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# heat_api_openstack.heat.api
input(type="imfile"
  file="/var/log/containers/heat/heat_api.log"
  tag="openstack.heat.api"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# heat_engine_openstack.heat.engine
input(type="imfile"
  file="/var/log/containers/heat/heat-engine.log"
  tag="openstack.heat.engine"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# keystone_openstack.keystone
input(type="imfile"
  file="/var/log/containers/keystone/keystone.log"
  tag="openstack.keystone"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# neutron_api_openstack.neutron.api
input(type="imfile"
  file="/var/log/containers/neutron/server.log"
  tag="openstack.neutron.api"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# nova_api_openstack.nova.api
input(type="imfile"
  file="/var/log/containers/nova/nova-api.log"
  tag="openstack.nova.api"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# nova_conductor_openstack.nova.conductor
input(type="imfile"
  file="/var/log/containers/nova/nova-conductor.log"
  tag="openstack.nova.conductor"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# nova_metadata_openstack.nova.api.metadata
input(type="imfile"
  file="/var/log/containers/nova/nova-metadata-api.log"
  tag="openstack.nova.api.metadata"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# nova_scheduler_openstack.nova.scheduler
input(type="imfile"
  file="/var/log/containers/nova/nova-scheduler.log"
  tag="openstack.nova.scheduler"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# nova_vnc_proxy_openstack.nova.vncproxy
input(type="imfile"
  file="/var/log/containers/nova/nova-novncproxy.log"
  tag="openstack.nova.vncproxy"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)
# pacemaker_system.pacemaker
input(type="imfile"
  file="/var/log/host/pacemaker.log"
  startmsg.regex="^[a-zA-Z]{3} [0-9]{2} [:0-9]{8}"
  tag="system.pacemaker"
)

template (name="rsyslog-node-index" type="list"
)
{
    constant(value="controller-0.redhat.local-" )
    property(dateFormat="year" name="timereported" )
    constant(value="." )
    property(dateFormat="month" name="timereported" )
    constant(value="." )
    property(dateFormat="day" name="timereported" )
}

template (name="rsyslog-record" type="list"
 option.jsonf="on")
{
    property(format="jsonf" name="timereported" outname="@timestamp" )
    property(format="jsonf" name="hostname" outname="host" )
    property(format="jsonf" name="syslogseverity-text" outname="severity" )
    property(format="jsonf" name="syslogfacility-text" outname="facility" )
    property(format="jsonf" name="syslogtag" outname="tag" )
    property(format="jsonf" name="app-name" outname="source" )
    property(format="jsonf" name="msg" outname="message" )
}
# elasticsearch
action(type="omelasticsearch"
    name="elasticsearch"
    dynSearchIndex="on"
    errorfile="/var/log/rsyslog/omelasticsearch.log"
    pwd="elastic"
    searchIndex="rsyslog-node-index"
    server="https://store-openstack-logging.apps.stftest1.lab"
    serverport="443"
    template="rsyslog-record"
    uid="elastic"
    tls.cacert="/etc/rsyslog.d/es-ca-cert.crt"
    tls.mycert="/etc/rsyslog.d/es-client-cert.pem"
    tls.myprivkey="/etc/rsyslog.d/es-client-key.pem"
  )

```

## compute-0

```
# cat /etc/rsyslog.d/50_openstack_logs.conf 
module(load="imfile")
module(load="omelasticsearch")
# nova_compute_openstack.nova.compute
input(type="imfile"
  file="/var/log/containers/nova/nova-compute.log"
  tag="openstack.nova.compute"
  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
)

template (name="rsyslog-node-index" type="list"
)
{
    constant(value="compute-0.redhat.local-" )
    property(dateFormat="year" name="timereported" )
    constant(value="." )
    property(dateFormat="month" name="timereported" )
    constant(value="." )
    property(dateFormat="day" name="timereported" )
}

template (name="rsyslog-record" type="list"
 option.jsonf="on")
{
    property(format="jsonf" name="timereported" outname="@timestamp" )
    property(format="jsonf" name="hostname" outname="host" )
    property(format="jsonf" name="syslogseverity-text" outname="severity" )
    property(format="jsonf" name="syslogfacility-text" outname="facility" )
    property(format="jsonf" name="syslogtag" outname="tag" )
    property(format="jsonf" name="app-name" outname="source" )
    property(format="jsonf" name="msg" outname="message" )
}
# elasticsearch
action(type="omelasticsearch"
    name="elasticsearch"
    dynSearchIndex="on"
    errorfile="/var/log/rsyslog/omelasticsearch.log"
    pwd="elastic"
    searchIndex="rsyslog-node-index"
    server="https://store-openstack-logging.apps.stftest1.lab"
    serverport="443"
    template="rsyslog-record"
    uid="elastic"
    tls.cacert="/etc/rsyslog.d/es-ca-cert.crt"
    tls.mycert="/etc/rsyslog.d/es-client-cert.pem"
    tls.myprivkey="/etc/rsyslog.d/es-client-key.pem"
  )
```

fixed

According to our records, this should be resolved by openstack-tripleo-heat-templates-11.3.2-1.20200914170177.el8ost.  This build is available now.