Attachment 1304590 Details for Bug 1466695 – es dc info

es dc info

es_log.txt (text/plain), 22.39 KB, created by Junqi Zhao on 2017-07-26 05:38:56 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Junqi Zhao

Created: 2017-07-26 05:38:56 UTC

Size: 22.39 KB

patch

obsolete

>**************************************************************************************************************************************************************
>Attach hostpath pv to es and es-ops,nodeSelector:logging-es-node: "1" 
>**************************************************************************************************************************************************************
>
># oc get dc
>NAME                      REVISION   DESIRED   CURRENT   TRIGGERED BY
>logging-curator           1          1         1         config
>logging-curator-ops       1          1         1         config
>logging-es-66jotb1g       4          1         1         config
>logging-es-ops-0wx0vzag   4          1         1         config
>logging-kibana            1          1         1         config
>logging-kibana-ops        1          1         1         config
>****************************************************************************************************************************************
># oc get dc logging-es-66jotb1g -o yaml
>apiVersion: v1
>kind: DeploymentConfig
>metadata:
>  creationTimestamp: 2017-07-26T03:17:38Z
>  generation: 11
>  labels:
>    component: es
>    deployment: logging-es-66jotb1g
>    logging-infra: elasticsearch
>    provider: openshift
>  name: logging-es-66jotb1g
>  namespace: logging
>  resourceVersion: "61807"
>  selfLink: /oapi/v1/namespaces/logging/deploymentconfigs/logging-es-66jotb1g
>  uid: fa037b8d-71b0-11e7-96a4-fa163eaa5651
>spec:
>  replicas: 1
>  selector:
>    component: es
>    deployment: logging-es-66jotb1g
>    logging-infra: elasticsearch
>    provider: openshift
>  strategy:
>    activeDeadlineSeconds: 21600
>    recreateParams:
>      timeoutSeconds: 600
>    resources: {}
>    type: Recreate
>  template:
>    metadata:
>      creationTimestamp: null
>      labels:
>        component: es
>        deployment: logging-es-66jotb1g
>        logging-infra: elasticsearch
>        provider: openshift
>      name: logging-es-66jotb1g
>    spec:
>      containers:
>      - env:
>        - name: NAMESPACE
>          valueFrom:
>            fieldRef:
>              apiVersion: v1
>              fieldPath: metadata.namespace
>        - name: KUBERNETES_TRUST_CERT
>          value: "true"
>        - name: SERVICE_DNS
>          value: logging-es-cluster
>        - name: CLUSTER_NAME
>          value: logging-es
>        - name: INSTANCE_RAM
>          value: 8Gi
>        - name: NODE_QUORUM
>          value: "1"
>        - name: RECOVER_AFTER_NODES
>          value: "0"
>        - name: RECOVER_EXPECTED_NODES
>          value: "1"
>        - name: RECOVER_AFTER_TIME
>          value: 5m
>        image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift3/logging-elasticsearch:v3.5
>        imagePullPolicy: Always
>        name: elasticsearch
>        ports:
>        - containerPort: 9200
>          name: restapi
>          protocol: TCP
>        - containerPort: 9300
>          name: cluster
>          protocol: TCP
>        resources:
>          limits:
>            memory: 8Gi
>          requests:
>            memory: 512Mi
>        securityContext:
>          privileged: true
>        terminationMessagePath: /dev/termination-log
>        volumeMounts:
>        - mountPath: /etc/elasticsearch/secret
>          name: elasticsearch
>          readOnly: true
>        - mountPath: /usr/share/java/elasticsearch/config
>          name: elasticsearch-config
>          readOnly: true
>        - mountPath: /elasticsearch/persistent
>          name: elasticsearch-storage
>      dnsPolicy: ClusterFirst
>      nodeSelector:
>        logging-es-node: "1"
>      restartPolicy: Always
>      securityContext:
>        supplementalGroups:
>        - 65534
>      serviceAccount: aggregated-logging-elasticsearch
>      serviceAccountName: aggregated-logging-elasticsearch
>      terminationGracePeriodSeconds: 30
>      volumes:
>      - name: elasticsearch
>        secret:
>          defaultMode: 420
>          secretName: logging-elasticsearch
>      - configMap:
>          defaultMode: 420
>          name: logging-elasticsearch
>        name: elasticsearch-config
>      - hostPath:
>          path: /usr/local/es-storage
>        name: elasticsearch-storage
>  test: false
>  triggers:
>  - type: ConfigChange
>status:
>  availableReplicas: 0
>  conditions:
>  - lastTransitionTime: 2017-07-26T03:17:38Z
>    lastUpdateTime: 2017-07-26T03:17:38Z
>    message: Deployment config does not have minimum availability.
>    status: "False"
>    type: Available
>  - lastTransitionTime: 2017-07-26T03:24:43Z
>    lastUpdateTime: 2017-07-26T03:24:43Z
>    message: replication controller "logging-es-66jotb1g-4" successfully rolled out
>    reason: NewReplicationControllerAvailable
>    status: "True"
>    type: Progressing
>  details:
>    causes:
>    - type: ConfigChange
>    message: config change
>  latestVersion: 4
>  observedGeneration: 11
>  readyReplicas: 1
>  replicas: 1
>  unavailableReplicas: 1
>  updatedReplicas: 1
>****************************************************************************************************************************************
># oc get dc logging-es-ops-0wx0vzag -o yaml 
>apiVersion: v1
>kind: DeploymentConfig
>metadata:
>  creationTimestamp: 2017-07-26T03:17:36Z
>  generation: 11
>  labels:
>    component: es-ops
>    deployment: logging-es-ops-0wx0vzag
>    logging-infra: elasticsearch
>    provider: openshift
>  name: logging-es-ops-0wx0vzag
>  namespace: logging
>  resourceVersion: "61815"
>  selfLink: /oapi/v1/namespaces/logging/deploymentconfigs/logging-es-ops-0wx0vzag
>  uid: f955f0a1-71b0-11e7-96a4-fa163eaa5651
>spec:
>  replicas: 1
>  selector:
>    component: es-ops
>    deployment: logging-es-ops-0wx0vzag
>    logging-infra: elasticsearch
>    provider: openshift
>  strategy:
>    activeDeadlineSeconds: 21600
>    recreateParams:
>      timeoutSeconds: 600
>    resources: {}
>    type: Recreate
>  template:
>    metadata:
>      creationTimestamp: null
>      labels:
>        component: es-ops
>        deployment: logging-es-ops-0wx0vzag
>        logging-infra: elasticsearch
>        provider: openshift
>      name: logging-es-ops-0wx0vzag
>    spec:
>      containers:
>      - env:
>        - name: NAMESPACE
>          valueFrom:
>            fieldRef:
>              apiVersion: v1
>              fieldPath: metadata.namespace
>        - name: KUBERNETES_TRUST_CERT
>          value: "true"
>        - name: SERVICE_DNS
>          value: logging-es-ops-cluster
>        - name: CLUSTER_NAME
>          value: logging-es-ops
>        - name: INSTANCE_RAM
>          value: 8Gi
>        - name: NODE_QUORUM
>          value: "1"
>        - name: RECOVER_AFTER_NODES
>          value: "0"
>        - name: RECOVER_EXPECTED_NODES
>          value: "1"
>        - name: RECOVER_AFTER_TIME
>          value: 5m
>        image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift3/logging-elasticsearch:v3.5
>        imagePullPolicy: Always
>        name: elasticsearch
>        ports:
>        - containerPort: 9200
>          name: restapi
>          protocol: TCP
>        - containerPort: 9300
>          name: cluster
>          protocol: TCP
>        resources:
>          limits:
>            memory: 8Gi
>          requests:
>            memory: 512Mi
>        securityContext:
>          privileged: true
>        terminationMessagePath: /dev/termination-log
>        volumeMounts:
>        - mountPath: /etc/elasticsearch/secret
>          name: elasticsearch
>          readOnly: true
>        - mountPath: /usr/share/java/elasticsearch/config
>          name: elasticsearch-config
>          readOnly: true
>        - mountPath: /elasticsearch/persistent
>          name: elasticsearch-storage
>      dnsPolicy: ClusterFirst
>      nodeSelector:
>        logging-es-node: "1"
>      restartPolicy: Always
>      securityContext:
>        supplementalGroups:
>        - 65534
>      serviceAccount: aggregated-logging-elasticsearch
>      serviceAccountName: aggregated-logging-elasticsearch
>      terminationGracePeriodSeconds: 30
>      volumes:
>      - name: elasticsearch
>        secret:
>          defaultMode: 420
>          secretName: logging-elasticsearch
>      - configMap:
>          defaultMode: 420
>          name: logging-elasticsearch
>        name: elasticsearch-config
>      - hostPath:
>          path: /usr/local/es-storage
>        name: elasticsearch-storage
>  test: false
>  triggers:
>  - type: ConfigChange
>status:
>  availableReplicas: 0
>  conditions:
>  - lastTransitionTime: 2017-07-26T03:17:36Z
>    lastUpdateTime: 2017-07-26T03:17:36Z
>    message: Deployment config does not have minimum availability.
>    status: "False"
>    type: Available
>  - lastTransitionTime: 2017-07-26T03:24:44Z
>    lastUpdateTime: 2017-07-26T03:24:44Z
>    message: replication controller "logging-es-ops-0wx0vzag-4" successfully rolled
>      out
>    reason: NewReplicationControllerAvailable
>    status: "True"
>    type: Progressing
>  details:
>    causes:
>    - type: ConfigChange
>    message: config change
>  latestVersion: 4
>  observedGeneration: 11
>  readyReplicas: 1
>  replicas: 1
>  unavailableReplicas: 1
>  updatedReplicas: 1
>****************************************************************************************************************************************
>Re run the ansible installation playbook,nodeSelector:logging-es-node: "1"  is missed from dc
>****************************************************************************************************************************************
>oc get dc
>NAME                      REVISION   DESIRED   CURRENT   TRIGGERED BY
>logging-curator           1          1         1         config
>logging-curator-ops       1          1         1         config
>logging-es-66jotb1g       5          1         1         config
>logging-es-ops-0wx0vzag   5          1         1         config
>logging-kibana            1          1         1         config
>logging-kibana-ops        1          1         1         config
>****************************************************************************************************************************************
># oc get dc logging-es-66jotb1g -o yaml
>apiVersion: v1
>kind: DeploymentConfig
>metadata:
>  creationTimestamp: 2017-07-26T03:17:38Z
>  generation: 14
>  labels:
>    component: es
>    deployment: logging-es-66jotb1g
>    logging-infra: elasticsearch
>    provider: openshift
>  name: logging-es-66jotb1g
>  namespace: logging
>  resourceVersion: "67035"
>  selfLink: /oapi/v1/namespaces/logging/deploymentconfigs/logging-es-66jotb1g
>  uid: fa037b8d-71b0-11e7-96a4-fa163eaa5651
>spec:
>  replicas: 1
>  selector:
>    component: es
>    deployment: logging-es-66jotb1g
>    logging-infra: elasticsearch
>    provider: openshift
>  strategy:
>    activeDeadlineSeconds: 21600
>    recreateParams:
>      timeoutSeconds: 600
>    resources: {}
>    type: Recreate
>  template:
>    metadata:
>      creationTimestamp: null
>      labels:
>        component: es
>        deployment: logging-es-66jotb1g
>        logging-infra: elasticsearch
>        provider: openshift
>      name: logging-es-66jotb1g
>    spec:
>      containers:
>      - env:
>        - name: NAMESPACE
>          valueFrom:
>            fieldRef:
>              apiVersion: v1
>              fieldPath: metadata.namespace
>        - name: KUBERNETES_TRUST_CERT
>          value: "true"
>        - name: SERVICE_DNS
>          value: logging-es-cluster
>        - name: CLUSTER_NAME
>          value: logging-es
>        - name: INSTANCE_RAM
>          value: 8Gi
>        - name: NODE_QUORUM
>          value: "1"
>        - name: RECOVER_AFTER_NODES
>          value: "0"
>        - name: RECOVER_EXPECTED_NODES
>          value: "1"
>        - name: RECOVER_AFTER_TIME
>          value: 5m
>        image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift3/logging-elasticsearch:v3.5
>        imagePullPolicy: Always
>        name: elasticsearch
>        ports:
>        - containerPort: 9200
>          name: restapi
>          protocol: TCP
>        - containerPort: 9300
>          name: cluster
>          protocol: TCP
>        resources:
>          limits:
>            memory: 8Gi
>          requests:
>            memory: 512Mi
>        terminationMessagePath: /dev/termination-log
>        volumeMounts:
>        - mountPath: /etc/elasticsearch/secret
>          name: elasticsearch
>          readOnly: true
>        - mountPath: /usr/share/java/elasticsearch/config
>          name: elasticsearch-config
>          readOnly: true
>        - mountPath: /elasticsearch/persistent
>          name: elasticsearch-storage
>      dnsPolicy: ClusterFirst
>      restartPolicy: Always
>      securityContext:
>        supplementalGroups:
>        - 65534
>      serviceAccount: aggregated-logging-elasticsearch
>      serviceAccountName: aggregated-logging-elasticsearch
>      terminationGracePeriodSeconds: 30
>      volumes:
>      - name: elasticsearch
>        secret:
>          defaultMode: 420
>          secretName: logging-elasticsearch
>      - configMap:
>          defaultMode: 420
>          name: logging-elasticsearch
>        name: elasticsearch-config
>      - hostPath:
>          path: /usr/local/es-storage
>        name: elasticsearch-storage
>  test: false
>  triggers:
>  - type: ConfigChange
>status:
>  availableReplicas: 0
>  conditions:
>  - lastTransitionTime: 2017-07-26T03:17:38Z
>    lastUpdateTime: 2017-07-26T03:17:38Z
>    message: Deployment config does not have minimum availability.
>    status: "False"
>    type: Available
>  - lastTransitionTime: 2017-07-26T03:30:21Z
>    lastUpdateTime: 2017-07-26T03:30:21Z
>    message: replication controller "logging-es-66jotb1g-5" successfully rolled out
>    reason: NewReplicationControllerAvailable
>    status: "True"
>    type: Progressing
>  details:
>    causes:
>    - type: ConfigChange
>    message: config change
>  latestVersion: 5
>  observedGeneration: 14
>  replicas: 1
>  unavailableReplicas: 1
>  updatedReplicas: 1
>****************************************************************************************************************************************
># oc get dc logging-es-ops-0wx0vzag -o yaml
>apiVersion: v1
>kind: DeploymentConfig
>metadata:
>  creationTimestamp: 2017-07-26T03:17:36Z
>  generation: 14
>  labels:
>    component: es-ops
>    deployment: logging-es-ops-0wx0vzag
>    logging-infra: elasticsearch
>    provider: openshift
>  name: logging-es-ops-0wx0vzag
>  namespace: logging
>  resourceVersion: "67006"
>  selfLink: /oapi/v1/namespaces/logging/deploymentconfigs/logging-es-ops-0wx0vzag
>  uid: f955f0a1-71b0-11e7-96a4-fa163eaa5651
>spec:
>  replicas: 1
>  selector:
>    component: es-ops
>    deployment: logging-es-ops-0wx0vzag
>    logging-infra: elasticsearch
>    provider: openshift
>  strategy:
>    activeDeadlineSeconds: 21600
>    recreateParams:
>      timeoutSeconds: 600
>    resources: {}
>    type: Recreate
>  template:
>    metadata:
>      creationTimestamp: null
>      labels:
>        component: es-ops
>        deployment: logging-es-ops-0wx0vzag
>        logging-infra: elasticsearch
>        provider: openshift
>      name: logging-es-ops-0wx0vzag
>    spec:
>      containers:
>      - env:
>        - name: NAMESPACE
>          valueFrom:
>            fieldRef:
>              apiVersion: v1
>              fieldPath: metadata.namespace
>        - name: KUBERNETES_TRUST_CERT
>          value: "true"
>        - name: SERVICE_DNS
>          value: logging-es-ops-cluster
>        - name: CLUSTER_NAME
>          value: logging-es-ops
>        - name: INSTANCE_RAM
>          value: 8Gi
>        - name: NODE_QUORUM
>          value: "1"
>        - name: RECOVER_AFTER_NODES
>          value: "0"
>        - name: RECOVER_EXPECTED_NODES
>          value: "1"
>        - name: RECOVER_AFTER_TIME
>          value: 5m
>        image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/openshift3/logging-elasticsearch:v3.5
>        imagePullPolicy: Always
>        name: elasticsearch
>        ports:
>        - containerPort: 9200
>          name: restapi
>          protocol: TCP
>        - containerPort: 9300
>          name: cluster
>          protocol: TCP
>        resources:
>          limits:
>            memory: 8Gi
>          requests:
>            memory: 512Mi
>        terminationMessagePath: /dev/termination-log
>        volumeMounts:
>        - mountPath: /etc/elasticsearch/secret
>          name: elasticsearch
>          readOnly: true
>        - mountPath: /usr/share/java/elasticsearch/config
>          name: elasticsearch-config
>          readOnly: true
>        - mountPath: /elasticsearch/persistent
>          name: elasticsearch-storage
>      dnsPolicy: ClusterFirst
>      restartPolicy: Always
>      securityContext:
>        supplementalGroups:
>        - 65534
>      serviceAccount: aggregated-logging-elasticsearch
>      serviceAccountName: aggregated-logging-elasticsearch
>      terminationGracePeriodSeconds: 30
>      volumes:
>      - name: elasticsearch
>        secret:
>          defaultMode: 420
>          secretName: logging-elasticsearch
>      - configMap:
>          defaultMode: 420
>          name: logging-elasticsearch
>        name: elasticsearch-config
>      - hostPath:
>          path: /usr/local/es-storage
>        name: elasticsearch-storage
>  test: false
>  triggers:
>  - type: ConfigChange
>status:
>  availableReplicas: 0
>  conditions:
>  - lastTransitionTime: 2017-07-26T03:17:36Z
>    lastUpdateTime: 2017-07-26T03:17:36Z
>    message: Deployment config does not have minimum availability.
>    status: "False"
>    type: Available
>  - lastTransitionTime: 2017-07-26T03:30:24Z
>    lastUpdateTime: 2017-07-26T03:30:24Z
>    message: replication controller "logging-es-ops-0wx0vzag-5" successfully rolled
>      out
>    reason: NewReplicationControllerAvailable
>    status: "True"
>    type: Progressing
>  details:
>    causes:
>    - type: ConfigChange
>    message: config change
>  latestVersion: 5
>  observedGeneration: 14
>  replicas: 1
>  unavailableReplicas: 1
>  updatedReplicas: 1
>****************************************************************************************************************************************
># oc get po
>NAME                              READY     STATUS             RESTARTS   AGE
>logging-curator-1-rg1wp           1/1       Running            0          2h
>logging-curator-ops-1-9dqh3       1/1       Running            0          2h
>logging-es-66jotb1g-5-g6sgn       0/1       CrashLoopBackOff   25         1h
>logging-es-ops-0wx0vzag-5-bpb12   0/1       CrashLoopBackOff   25         1h
>logging-fluentd-jr7qw             1/1       Running            0          1h
>logging-fluentd-px530             1/1       Running            0          1h
>logging-kibana-1-4ch1g            2/2       Running            0          2h
>logging-kibana-ops-1-5nn62        2/2       Running            0          2h
>****************************************************************************************************************************************
># oc logs logging-es-66jotb1g-5-g6sgn
>[2017-07-26 05:14:59,598][INFO ][container.run            ] Begin Elasticsearch startup script
>[2017-07-26 05:14:59,602][INFO ][container.run            ] Comparing the specified RAM to the maximum recommended for Elasticsearch...
>[2017-07-26 05:14:59,603][INFO ][container.run            ] Inspecting the maximum RAM available...
>[2017-07-26 05:14:59,605][INFO ][container.run            ] ES_JAVA_OPTS: '-Dmapper.allow_dots_in_name=true -Xms4096m -Xmx4096m'
>Exception in thread "main" java.lang.IllegalStateException: Failed to obtain node lock, is the following location writable?: [/elasticsearch/persistent/logging-es/data/logging-es]
>Likely root cause: java.nio.file.AccessDeniedException: /elasticsearch/persistent/logging-es/data/logging-es/nodes/0/node.lock
>	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:84)
>	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
>	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
>	at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
>	at java.nio.channels.FileChannel.open(FileChannel.java:287)
>	at java.nio.channels.FileChannel.open(FileChannel.java:335)
>	at org.apache.lucene.store.NativeFSLockFactory.obtainFSLock(NativeFSLockFactory.java:113)
>	at org.apache.lucene.store.FSLockFactory.obtainLock(FSLockFactory.java:41)
>	at org.apache.lucene.store.BaseDirectory.obtainLock(BaseDirectory.java:45)
>	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:174)
>	at org.elasticsearch.node.Node.<init>(Node.java:165)
>	at org.elasticsearch.node.Node.<init>(Node.java:140)
>	at org.elasticsearch.node.NodeBuilder.build(NodeBuilder.java:143)
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:194)
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:286)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:45)
>Refer to the log for complete error details.
>****************************************************************************************************************************************
># oc logs logging-es-ops-0wx0vzag-5-bpb12
>[2017-07-26 05:19:44,624][INFO ][container.run            ] Begin Elasticsearch startup script
>[2017-07-26 05:19:44,627][INFO ][container.run            ] Comparing the specified RAM to the maximum recommended for Elasticsearch...
>[2017-07-26 05:19:44,629][INFO ][container.run            ] Inspecting the maximum RAM available...
>[2017-07-26 05:19:44,631][INFO ][container.run            ] ES_JAVA_OPTS: '-Dmapper.allow_dots_in_name=true -Xms4096m -Xmx4096m'
>Exception in thread "main" java.lang.IllegalStateException: Failed to obtain node lock, is the following location writable?: [/elasticsearch/persistent/logging-es-ops/data/logging-es-ops]
>Likely root cause: java.nio.file.AccessDeniedException: /elasticsearch/persistent/logging-es-ops/data/logging-es-ops/nodes/0/node.lock
>	at sun.nio.fs.UnixException.translateToIOException(UnixException.java:84)
>	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
>	at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
>	at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
>	at java.nio.channels.FileChannel.open(FileChannel.java:287)
>	at java.nio.channels.FileChannel.open(FileChannel.java:335)
>	at org.apache.lucene.store.NativeFSLockFactory.obtainFSLock(NativeFSLockFactory.java:113)
>	at org.apache.lucene.store.FSLockFactory.obtainLock(FSLockFactory.java:41)
>	at org.apache.lucene.store.BaseDirectory.obtainLock(BaseDirectory.java:45)
>	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:174)
>	at org.elasticsearch.node.Node.<init>(Node.java:165)
>	at org.elasticsearch.node.Node.<init>(Node.java:140)
>	at org.elasticsearch.node.NodeBuilder.build(NodeBuilder.java:143)
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:194)
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:286)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:45)
>Refer to the log for complete error details.
>
>
>****************************************************************************************************************************************
>

Actions: View

Attachments on bug 1466695: 1304143 | 1304590