Hide Forgot
Description of problem: I updated the "instance" custom resource which defines my cluster-logging setup. However, the elasticsearch deployments don't seem to get updated with the new configs. Additionally, if you delete one of those deployments (I was hoping it would be recreated with the correct config), the deployment doesn't get recreated. Version-Release number of selected component (if applicable): 4.1 How reproducible: Happened once, haven't verified if it's easily reproducible Steps to Reproduce: 1. Edit "instance" in the web console Actual results: $ oc get pod NAME READY STATUS RESTARTS AGE cluster-logging-operator-5d855886bc-5l4c5 1/1 Running 0 5m47s elasticsearch-cdm-4soi24zg-1-74f74c8bc9-9ndfd 0/2 ContainerCreating 0 12m elasticsearch-cdm-4soi24zg-3-56599b45f9-fv8x9 0/2 ContainerCreating 0 12m Deployment appears to be paused: $ oc get deployment elasticsearch-cdm-4soi24zg-1 -o yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: annotations: deployment.kubernetes.io/revision: "2" creationTimestamp: "2019-07-17T18:23:44Z" generation: 5 labels: cluster-name: elasticsearch component: elasticsearch es-node-client: "true" es-node-data: "true" es-node-master: "true" node-name: elasticsearch-cdm-4soi24zg-1 tuned.openshift.io/elasticsearch: "true" name: elasticsearch-cdm-4soi24zg-1 namespace: openshift-logging ownerReferences: - apiVersion: logging.openshift.io/v1 controller: true kind: Elasticsearch name: elasticsearch uid: 00756a94-a8c0-11e9-93f2-0a5e8b8252ca resourceVersion: "66982" selfLink: /apis/extensions/v1beta1/namespaces/openshift-logging/deployments/elasticsearch-cdm-4soi24zg-1 uid: 02de1634-a8c0-11e9-b9d3-024ccf00d6b0 spec: paused: true progressDeadlineSeconds: 1800 replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: cluster-name: elasticsearch es-node-client: "true" es-node-data: "true" es-node-master: "true" node-name: elasticsearch-cdm-4soi24zg-1 strategy: type: Recreate template: metadata: creationTimestamp: null labels: cluster-name: elasticsearch component: elasticsearch es-node-client: "true" es-node-data: "true" es-node-master: "true" node-name: elasticsearch-cdm-4soi24zg-1 tuned.openshift.io/elasticsearch: "true" spec: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: es-node-client operator: In values: - "true" - key: es-node-data operator: In values: - "true" - key: es-node-master operator: In values: - "true" topologyKey: kubernetes.io/hostname weight: 100 containers: - env: - name: DC_NAME value: elasticsearch-cdm-4soi24zg-1 - name: NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: KUBERNETES_TRUST_CERT value: "true" - name: SERVICE_DNS value: elasticsearch-cluster - name: CLUSTER_NAME value: elasticsearch - name: INSTANCE_RAM value: 16Gi - name: HEAP_DUMP_LOCATION value: /elasticsearch/persistent/heapdump.hprof - name: RECOVER_AFTER_TIME value: 5m - name: READINESS_PROBE_TIMEOUT value: "30" - name: POD_LABEL value: cluster=elasticsearch - name: IS_MASTER value: "true" - name: HAS_DATA value: "true" image: registry.redhat.io/openshift4/ose-logging-elasticsearch5:v4.1.4-201906271212 imagePullPolicy: IfNotPresent name: elasticsearch ports: - containerPort: 9300 name: cluster protocol: TCP - containerPort: 9200 name: restapi protocol: TCP readinessProbe: exec: command: - /usr/share/elasticsearch/probe/readiness.sh failureThreshold: 3 initialDelaySeconds: 10 periodSeconds: 5 successThreshold: 1 timeoutSeconds: 30 resources: limits: cpu: "4" memory: 4Gi requests: cpu: 100m memory: 1Gi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /elasticsearch/persistent name: elasticsearch-storage - mountPath: /usr/share/java/elasticsearch/config name: elasticsearch-config - mountPath: /etc/openshift/elasticsearch/secret name: certificates - args: - --https-address=:60000 - --provider=openshift - --upstream=https://127.0.0.1:9200 - --tls-cert=/etc/proxy/secrets/tls.crt - --tls-key=/etc/proxy/secrets/tls.key - --upstream-ca=/etc/proxy/elasticsearch/admin-ca - --openshift-service-account=elasticsearch - '-openshift-sar={"resource": "namespaces", "verb": "get"}' - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}}' - --pass-user-bearer-token - --cookie-secret=GM02fd+3zCaAGR3tdvC+7w== image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.1.4-201906271212 imagePullPolicy: IfNotPresent name: proxy ports: - containerPort: 60000 name: metrics protocol: TCP resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /etc/proxy/secrets name: elasticsearch-metrics - mountPath: /etc/proxy/elasticsearch name: certificates dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: elasticsearch serviceAccountName: elasticsearch terminationGracePeriodSeconds: 30 tolerations: - effect: NoSchedule key: node.kubernetes.io/disk-pressure operator: Exists volumes: - configMap: defaultMode: 420 name: elasticsearch name: elasticsearch-config - name: elasticsearch-storage persistentVolumeClaim: claimName: elasticsearch-elasticsearch-cdm-4soi24zg-1 - name: certificates secret: defaultMode: 420 secretName: elasticsearch - name: elasticsearch-metrics secret: defaultMode: 420 secretName: elasticsearch-metrics status: conditions: - lastTransitionTime: "2019-07-17T18:23:44Z" lastUpdateTime: "2019-07-17T18:23:44Z" message: Deployment does not have minimum availability. reason: MinimumReplicasUnavailable status: "False" type: Available - lastTransitionTime: "2019-07-17T18:29:19Z" lastUpdateTime: "2019-07-17T18:29:19Z" message: Deployment is paused reason: DeploymentPaused status: Unknown type: Progressing observedGeneration: 5 replicas: 1 unavailableReplicas: 1 updatedReplicas: 1 $ oc logs cluster-logging-operator-5d855886bc-5l4c5 time="2019-07-17T20:15:36Z" level=info msg="Go Version: go1.10.8" time="2019-07-17T20:15:36Z" level=info msg="Go OS/Arch: linux/amd64" time="2019-07-17T20:15:36Z" level=info msg="operator-sdk Version: 0.0.7" time="2019-07-17T20:15:36Z" level=info msg="Watching logging.openshift.io/v1, ClusterLogging, openshift-logging, 5000000000" time="2019-07-17T20:15:43Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:15:45Z" level=error msg="Unable to read file to get contents: open /tmp/_working_dir/kibana-proxy-oauth.secret: no such file or directory" time="2019-07-17T20:16:02Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:16:21Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:16:22Z" level=info msg="Updating status of Elasticsearch" time="2019-07-17T20:16:41Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:17:00Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:17:20Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:17:39Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:17:58Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:18:18Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:18:37Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:18:57Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:19:16Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:19:35Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:19:55Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:20:14Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:20:34Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:20:53Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch" time="2019-07-17T20:21:12Z" level=info msg="Elasticsearch node configuration change found, updating elasticsearch"
*** This bug has been marked as a duplicate of bug 1746542 ***