Bug 1767111

https://github.com/openshift/openshift-ansible/pull/11991

@mburke - we'll need to add this to the docs for 3.11

Verified. Result as following

1. The default Env values in DS
        - env:
          - name: TRANSFORM_EVENTS
            value: "true"
          - name: MERGE_JSON_LOG
            value: "true"

2. 
1) Customized in ansible
openshift_logging_fluentd_extra_keep_fields="StringNumber,Number"
openshift_logging_fluentd_keep_empty_fields="space2"
openshift_logging_fluentd_undefined_name="anliskip"
openshift_logging_fluentd_undefined_to_string="true"
openshift_logging_fluentd_undefined_dot_replace_char="_"
openshift_logging_fluentd_undefined_max_num_fields=5
openshift_logging_fluentd_use_undefined="true"
openshift_logging_fluentd_use_multiline_json="true"
openshift_logging_fluentd_use_multiline_journal="true"
openshift_logging_install_eventrouter=True
openshift_logging_es_allow_external=True
2) Env in DS
          - name: MERGE_JSON_LOG
            value: "true"
          - name: CDM_EXTRA_KEEP_FIELDS
            value: StringNumber,Number
          - name: CDM_KEEP_EMPTY_FIELDS
            value: space2
          - name: CDM_USE_UNDEFINED
            value: "true"
          - name: CDM_UNDEFINED_NAME
            value: anliskip
          - name: CDM_UNDEFINED_TO_STRING
            value: "true"
          - name: CDM_UNDEFINED_DOT_REPLACE_CHAR
            value: _
          - name: CDM_UNDEFINED_MAX_NUM_FIELDS
            value: "5"
          - name: USE_MULTILINE_JSON
            value: "true"
          - name: USE_MULTILINE_JOURNAL
            value: "true"

3). Json handling in ES. 
3.1.) A json logs
      {
        "_index" : "project.logjsonx2.42786973-051b-11ea-a982-fa163e9fc3ac.2019.11.12",
        "_type" : "com.redhat.viaq.common",
        "_id" : "NzBiOTVhNDYtMGJjOC00ODk2LTgzMTYtNzkzZjFkYTA3NTAw",
        "_score" : 1.0,
        "_source" : {
          "message" : "MERGE_JSON_LOG=true",
          "level" : "debug",
          "StringNumber" : "10",
          "Number" : 10,
          "space2" : "",
          "docker" : {
            "container_id" : "b5a9941676ea6868afa204beb37d634eae62beefccd3411f68c0a915b30cd176"
          },
          "kubernetes" : {
            "container_name" : "centos-logtest",
            "namespace_name" : "logjsonx2",
            "pod_name" : "centos-logtest-x79pq",
            "container_image" : "docker.io/mffiedler/ocp-logtest:latest",
            "container_image_id" : "docker-pullable://docker.io/mffiedler/ocp-logtest@sha256:f23bea6f669d125f2f317e3097a0a4da48e8792746db32838725b45efa6c64a4",
            "pod_id" : "1cc7cd6a-051e-11ea-a982-fa163e9fc3ac",
            "labels" : {
              "run" : "centos-logtest",
              "test" : "centos-logtest"
            },
            "host" : "qe-anli311node-1",
            "master_url" : "https://kubernetes.default.svc.cluster.local",
            "namespace_id" : "42786973-051b-11ea-a982-fa163e9fc3ac"
          },
          "hostname" : "qe-anli311node-1",
          "pipeline_metadata" : {
            "collector" : {
              "ipaddr4" : "10.130.0.15",
              "ipaddr6" : "fe80::54b3:2cff:fe1c:886",
              "inputname" : "fluent-plugin-systemd",
              "name" : "fluentd",
              "received_at" : "2019-11-12T08:00:27.323806+00:00",
              "version" : "0.12.43 1.6.0"
            }
          },
          "anliskip" : "{\"Layer1\":\"layer1 0\",\"layer2\":{\"name\":\"Layer2 1\",\"tips\":\"Decide by PRESERVE_JSON_LOG\"},\"foo.bar\":\"Dot Item\",\"{foobar}\":\"Brace Item\",\"[foobar]\":\"Bracket Item\",\"foo:bar\":\"Colon Item\",\"foo bar\":\"Space Item\",\"space1\":\"\",\"space3\":\"\"}",
          "@timestamp" : "2019-11-12T08:00:27.297793+00:00",
          "viaq_msg_id" : "NzBiOTVhNDYtMGJjOC00ODk2LTgzMTYtNzkzZjFkYTA3NTAw"
        }
      }
    ]

3.1.2) the kube-eventrouter logs

    "kubernetes": {
      "container_name": "kube-eventrouter",
      "namespace_name": "default",
      "pod_name": "logging-eventrouter-1-cppfj",
      "container_image": "registry.access.stage.redhat.com/openshift3/ose-logging-eventrouter:v3.11.154",
      "container_image_id": "docker-pullable://registry.access.stage.redhat.com/openshift3/ose-logging-eventrouter@sha256:3b5b3a17ef0a86c3e013e0c60d72874a34fc220dea872bcb6dd22d6fe6847628",
      "pod_id": "370bfff5-0523-11ea-a982-fa163e9fc3ac",
      "labels": {
        "component": "eventrouter",
        "deployment": "logging-eventrouter-1",
        "deploymentconfig": "logging-eventrouter",
        "logging-infra": "eventrouter",
        "provider": "openshift"
      },
      "host": "qe-anli311node-1",
      "master_url": "https://kubernetes.default.svc.cluster.local",
      "namespace_id": "80920659-0510-11ea-a5ea-fa163e9fc3ac",
      "event": {
        "metadata": {
          "name": "logging-eventrouter-1-cppfj.15d65bb294c667b3",
          "namespace": "default",
          "selfLink": "/api/v1/namespaces/default/events/logging-eventrouter-1-cppfj.15d65bb294c667b3",
          "uid": "38a74566-0523-11ea-a982-fa163e9fc3ac",
          "resourceVersion": "21468"
        },
        "involvedObject": {
          "kind": "Pod",
          "namespace": "default",
          "name": "logging-eventrouter-1-cppfj",
          "uid": "370bfff5-0523-11ea-a982-fa163e9fc3ac",
          "apiVersion": "v1",
          "resourceVersion": "21449",
          "fieldPath": "spec.containers{kube-eventrouter}"
        },
        "reason": "Pulled",
        "source": {
          "component": "kubelet",
          "host": "qe-anli311node-1"
        },
        "firstTimestamp": "2019-11-12T08:05:42Z",
        "lastTimestamp": "2019-11-12T08:05:42Z",
        "count": 1,
        "type": "Normal",
        "verb": "ADDED"
      }
    },

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2019:3817