Attachment 1447755 Details for Bug 1571641 – still shows "Connection refused error" for hawkular-cassandra pod

still shows "Connection refused error" for hawkular-cassandra pod

bz_1571641.txt (text/plain), 18.97 KB, created by Junqi Zhao on 2018-06-05 08:29:22 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Junqi Zhao

Created: 2018-06-05 08:29:22 UTC

Size: 18.97 KB

patch

obsolete

># oc get po -o wide
>NAME                            READY     STATUS      RESTARTS   AGE       IP            NODE
>hawkular-cassandra-1-kt858      1/1       Running     0          8m        10.129.0.26   qe-juzhao-310-qeos-1-nrr-1
>hawkular-metrics-gxwqq          1/1       Running     0          8m        10.128.0.25   qe-juzhao-310-qeos-1-master-etcd-1
>hawkular-metrics-schema-cltg2   0/1       Completed   0          9m        10.129.0.25   qe-juzhao-310-qeos-1-nrr-1
>heapster-b7rzw                  1/1       Running     0          8m        10.129.0.27   qe-juzhao-310-qeos-1-nrr-1
>**************************************************************************************************************************************
># curl 10.129.0.26:7575/metrics
>curl: (7) Failed connect to 10.129.0.26:7575; Connection refused
>**************************************************************************************************************************************
># curl 10.128.0.25:7575/metrics
># HELP jvm_threads_current Current thread count of a JVM
># TYPE jvm_threads_current gauge
>jvm_threads_current 146.0
># HELP jvm_threads_daemon Daemon thread count of a JVM
># TYPE jvm_threads_daemon gauge
>jvm_threads_daemon 23.0
># HELP jvm_threads_peak Peak thread count of a JVM
># TYPE jvm_threads_peak gauge
>jvm_threads_peak 204.0
># HELP jvm_threads_started_total Started thread count of a JVM
># TYPE jvm_threads_started_total counter
>jvm_threads_started_total 1884.0
># HELP jvm_threads_deadlocked Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers
># TYPE jvm_threads_deadlocked gauge
>jvm_threads_deadlocked 0.0
># HELP jvm_threads_deadlocked_monitor Cycles of JVM-threads that are in deadlock waiting to acquire object monitors
># TYPE jvm_threads_deadlocked_monitor gauge
>jvm_threads_deadlocked_monitor 0.0
># HELP jvm_gc_collection_seconds Time spent in a given JVM garbage collector in seconds.
># TYPE jvm_gc_collection_seconds summary
>jvm_gc_collection_seconds_count{gc="PS Scavenge",} 14.0
>jvm_gc_collection_seconds_sum{gc="PS Scavenge",} 0.95
>jvm_gc_collection_seconds_count{gc="PS MarkSweep",} 1.0
>jvm_gc_collection_seconds_sum{gc="PS MarkSweep",} 0.292
># HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
># TYPE process_cpu_seconds_total counter
>process_cpu_seconds_total 61.76
># HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
># TYPE process_start_time_seconds gauge
>process_start_time_seconds 1.528182918948E9
># HELP process_open_fds Number of open file descriptors.
># TYPE process_open_fds gauge
>process_open_fds 590.0
># HELP process_max_fds Maximum number of open file descriptors.
># TYPE process_max_fds gauge
>process_max_fds 1048576.0
># HELP process_virtual_memory_bytes Virtual memory size in bytes.
># TYPE process_virtual_memory_bytes gauge
>process_virtual_memory_bytes 5.43731712E9
># HELP process_resident_memory_bytes Resident memory size in bytes.
># TYPE process_resident_memory_bytes gauge
>process_resident_memory_bytes 9.02500352E8
># HELP jvm_memory_bytes_used Used bytes of a given JVM memory area.
># TYPE jvm_memory_bytes_used gauge
>jvm_memory_bytes_used{area="heap",} 2.55323432E8
>jvm_memory_bytes_used{area="nonheap",} 1.53772632E8
># HELP jvm_memory_bytes_committed Committed (bytes) of a given JVM memory area.
># TYPE jvm_memory_bytes_committed gauge
>jvm_memory_bytes_committed{area="heap",} 1.249378304E9
>jvm_memory_bytes_committed{area="nonheap",} 1.687552E8
># HELP jvm_memory_bytes_max Max (bytes) of a given JVM memory area.
># TYPE jvm_memory_bytes_max gauge
>jvm_memory_bytes_max{area="heap",} 1.249378304E9
>jvm_memory_bytes_max{area="nonheap",} 7.80140544E8
># HELP jvm_memory_pool_bytes_used Used bytes of a given JVM memory pool.
># TYPE jvm_memory_pool_bytes_used gauge
>jvm_memory_pool_bytes_used{pool="Code Cache",} 3.8289472E7
>jvm_memory_pool_bytes_used{pool="Metaspace",} 1.03141184E8
>jvm_memory_pool_bytes_used{pool="Compressed Class Space",} 1.2341976E7
>jvm_memory_pool_bytes_used{pool="PS Eden Space",} 1.87417624E8
>jvm_memory_pool_bytes_used{pool="PS Survivor Space",} 360448.0
>jvm_memory_pool_bytes_used{pool="PS Old Gen",} 6.754536E7
># HELP jvm_memory_pool_bytes_committed Committed bytes of a given JVM memory pool.
># TYPE jvm_memory_pool_bytes_committed gauge
>jvm_memory_pool_bytes_committed{pool="Code Cache",} 4.1091072E7
>jvm_memory_pool_bytes_committed{pool="Metaspace",} 1.12459776E8
>jvm_memory_pool_bytes_committed{pool="Compressed Class Space",} 1.5204352E7
>jvm_memory_pool_bytes_committed{pool="PS Eden Space",} 2.16530944E8
>jvm_memory_pool_bytes_committed{pool="PS Survivor Space",} 1.21110528E8
>jvm_memory_pool_bytes_committed{pool="PS Old Gen",} 9.11736832E8
># HELP jvm_memory_pool_bytes_max Max bytes of a given JVM memory pool.
># TYPE jvm_memory_pool_bytes_max gauge
>jvm_memory_pool_bytes_max{pool="Code Cache",} 2.5165824E8
>jvm_memory_pool_bytes_max{pool="Metaspace",} 2.68435456E8
>jvm_memory_pool_bytes_max{pool="Compressed Class Space",} 2.60046848E8
>jvm_memory_pool_bytes_max{pool="PS Eden Space",} 2.16530944E8
>jvm_memory_pool_bytes_max{pool="PS Survivor Space",} 1.21110528E8
>jvm_memory_pool_bytes_max{pool="PS Old Gen",} 9.11736832E8
># HELP jmx_config_reload_success_total Number of times configuration have successfully been reloaded.
># TYPE jmx_config_reload_success_total counter
>jmx_config_reload_success_total 0.0
># HELP jmx_scrape_duration_seconds Time this JMX scrape took, in seconds.
># TYPE jmx_scrape_duration_seconds gauge
>jmx_scrape_duration_seconds 5.81282E-4
># HELP jmx_scrape_error Non-zero if this scrape failed.
># TYPE jmx_scrape_error gauge
>jmx_scrape_error 0.0
># HELP jvm_classes_loaded The number of classes that are currently loaded in the JVM
># TYPE jvm_classes_loaded gauge
>jvm_classes_loaded 18257.0
># HELP jvm_classes_loaded_total The total number of classes that have been loaded since the JVM has started execution
># TYPE jvm_classes_loaded_total counter
>jvm_classes_loaded_total 18260.0
># HELP jvm_classes_unloaded_total The total number of classes that have been unloaded since the JVM has started execution
># TYPE jvm_classes_unloaded_total counter
>jvm_classes_unloaded_total 3.0
># HELP jmx_config_reload_failure_total Number of times configuration have failed to be reloaded.
># TYPE jmx_config_reload_failure_total counter
>jmx_config_reload_failure_total 0.0
># HELP jvm_info JVM version info
># TYPE jvm_info gauge
>jvm_info{version="1.8.0_171-b10",vendor="Oracle Corporation",} 1.0
># curl 10.128.0.25:7575/metrics
># HELP jvm_threads_current Current thread count of a JVM
># TYPE jvm_threads_current gauge
>jvm_threads_current 148.0
># HELP jvm_threads_daemon Daemon thread count of a JVM
># TYPE jvm_threads_daemon gauge
>jvm_threads_daemon 23.0
># HELP jvm_threads_peak Peak thread count of a JVM
># TYPE jvm_threads_peak gauge
>jvm_threads_peak 204.0
># HELP jvm_threads_started_total Started thread count of a JVM
># TYPE jvm_threads_started_total counter
>jvm_threads_started_total 1886.0
># HELP jvm_threads_deadlocked Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers
># TYPE jvm_threads_deadlocked gauge
>jvm_threads_deadlocked 0.0
># HELP jvm_threads_deadlocked_monitor Cycles of JVM-threads that are in deadlock waiting to acquire object monitors
># TYPE jvm_threads_deadlocked_monitor gauge
>jvm_threads_deadlocked_monitor 0.0
># HELP jvm_gc_collection_seconds Time spent in a given JVM garbage collector in seconds.
># TYPE jvm_gc_collection_seconds summary
>jvm_gc_collection_seconds_count{gc="PS Scavenge",} 14.0
>jvm_gc_collection_seconds_sum{gc="PS Scavenge",} 0.95
>jvm_gc_collection_seconds_count{gc="PS MarkSweep",} 1.0
>jvm_gc_collection_seconds_sum{gc="PS MarkSweep",} 0.292
># HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
># TYPE process_cpu_seconds_total counter
>process_cpu_seconds_total 62.01
># HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
># TYPE process_start_time_seconds gauge
>process_start_time_seconds 1.528182918948E9
># HELP process_open_fds Number of open file descriptors.
># TYPE process_open_fds gauge
>process_open_fds 590.0
># HELP process_max_fds Maximum number of open file descriptors.
># TYPE process_max_fds gauge
>process_max_fds 1048576.0
># HELP process_virtual_memory_bytes Virtual memory size in bytes.
># TYPE process_virtual_memory_bytes gauge
>process_virtual_memory_bytes 5.43731712E9
># HELP process_resident_memory_bytes Resident memory size in bytes.
># TYPE process_resident_memory_bytes gauge
>process_resident_memory_bytes 9.027584E8
># HELP jvm_memory_bytes_used Used bytes of a given JVM memory area.
># TYPE jvm_memory_bytes_used gauge
>jvm_memory_bytes_used{area="heap",} 2.59920848E8
>jvm_memory_bytes_used{area="nonheap",} 1.53920592E8
># HELP jvm_memory_bytes_committed Committed (bytes) of a given JVM memory area.
># TYPE jvm_memory_bytes_committed gauge
>jvm_memory_bytes_committed{area="heap",} 1.249378304E9
>jvm_memory_bytes_committed{area="nonheap",} 1.687552E8
># HELP jvm_memory_bytes_max Max (bytes) of a given JVM memory area.
># TYPE jvm_memory_bytes_max gauge
>jvm_memory_bytes_max{area="heap",} 1.249378304E9
>jvm_memory_bytes_max{area="nonheap",} 7.80140544E8
># HELP jvm_memory_pool_bytes_used Used bytes of a given JVM memory pool.
># TYPE jvm_memory_pool_bytes_used gauge
>jvm_memory_pool_bytes_used{pool="Code Cache",} 3.8394496E7
>jvm_memory_pool_bytes_used{pool="Metaspace",} 1.03180512E8
>jvm_memory_pool_bytes_used{pool="Compressed Class Space",} 1.2345584E7
>jvm_memory_pool_bytes_used{pool="PS Eden Space",} 1.9201504E8
>jvm_memory_pool_bytes_used{pool="PS Survivor Space",} 360448.0
>jvm_memory_pool_bytes_used{pool="PS Old Gen",} 6.754536E7
># HELP jvm_memory_pool_bytes_committed Committed bytes of a given JVM memory pool.
># TYPE jvm_memory_pool_bytes_committed gauge
>jvm_memory_pool_bytes_committed{pool="Code Cache",} 4.1091072E7
>jvm_memory_pool_bytes_committed{pool="Metaspace",} 1.12459776E8
>jvm_memory_pool_bytes_committed{pool="Compressed Class Space",} 1.5204352E7
>jvm_memory_pool_bytes_committed{pool="PS Eden Space",} 2.16530944E8
>jvm_memory_pool_bytes_committed{pool="PS Survivor Space",} 1.21110528E8
>jvm_memory_pool_bytes_committed{pool="PS Old Gen",} 9.11736832E8
># HELP jvm_memory_pool_bytes_max Max bytes of a given JVM memory pool.
># TYPE jvm_memory_pool_bytes_max gauge
>jvm_memory_pool_bytes_max{pool="Code Cache",} 2.5165824E8
>jvm_memory_pool_bytes_max{pool="Metaspace",} 2.68435456E8
>jvm_memory_pool_bytes_max{pool="Compressed Class Space",} 2.60046848E8
>jvm_memory_pool_bytes_max{pool="PS Eden Space",} 2.16530944E8
>jvm_memory_pool_bytes_max{pool="PS Survivor Space",} 1.21110528E8
>jvm_memory_pool_bytes_max{pool="PS Old Gen",} 9.11736832E8
># HELP jmx_config_reload_success_total Number of times configuration have successfully been reloaded.
># TYPE jmx_config_reload_success_total counter
>jmx_config_reload_success_total 0.0
># HELP jmx_scrape_duration_seconds Time this JMX scrape took, in seconds.
># TYPE jmx_scrape_duration_seconds gauge
>jmx_scrape_duration_seconds 1.75784E-4
># HELP jmx_scrape_error Non-zero if this scrape failed.
># TYPE jmx_scrape_error gauge
>jmx_scrape_error 0.0
># HELP jvm_classes_loaded The number of classes that are currently loaded in the JVM
># TYPE jvm_classes_loaded gauge
>jvm_classes_loaded 18262.0
># HELP jvm_classes_loaded_total The total number of classes that have been loaded since the JVM has started execution
># TYPE jvm_classes_loaded_total counter
>jvm_classes_loaded_total 18265.0
># HELP jvm_classes_unloaded_total The total number of classes that have been unloaded since the JVM has started execution
># TYPE jvm_classes_unloaded_total counter
>jvm_classes_unloaded_total 3.0
># HELP jmx_config_reload_failure_total Number of times configuration have failed to be reloaded.
># TYPE jmx_config_reload_failure_total counter
>jmx_config_reload_failure_total 0.0
># HELP jvm_info JVM version info
># TYPE jvm_info gauge
>jvm_info{version="1.8.0_171-b10",vendor="Oracle Corporation",} 1.0
>**************************************************************************************************************************************
># oc get rc hawkular-cassandra-1 -o yaml
>apiVersion: v1
>kind: ReplicationController
>metadata:
>  annotations:
>    kubectl.kubernetes.io/last-applied-configuration: |
>      {"apiVersion":"v1","kind":"ReplicationController","metadata":{"annotations":{},"creationTimestamp":"2018-06-05T07:13:22Z","generation":1,"labels":{"metrics-infra":"hawkular-cassandra","name":"hawkular-cassandra","type":"hawkular-cassandra"},"name":"hawkular-cassandra-1","namespace":"openshift-infra","selfLink":"/api/v1/namespaces/openshift-infra/replicationcontrollers/hawkular-cassandra-1","uid":"eeb819d5-688f-11e8-a8d7-fa163ee42f0d"},"spec":{"replicas":1,"selector":{"name":"hawkular-cassandra-1"},"template":{"metadata":{"creationTimestamp":null,"labels":{"metrics-infra":"hawkular-cassandra","name":"hawkular-cassandra-1","type":"hawkular-cassandra"}},"spec":{"containers":[{"command":["/opt/apache-cassandra/bin/cassandra-docker.sh","--cluster_name=hawkular-metrics","--data_volume=/cassandra_data","--internode_encryption=all","--require_node_auth=true","--enable_client_encryption=true","--require_client_auth=true"],"env":[{"name":"CASSANDRA_MASTER","value":"true"},{"name":"CASSANDRA_DATA_VOLUME","value":"/cassandra_data"},{"name":"JVM_OPTS","value":"-Dcassandra.commitlog.ignorereplayerrors=true"},{"name":"ENABLE_PROMETHEUS_ENDPOINT","value":"True"},{"name":"TRUSTSTORE_NODES_AUTHORITIES","value":"/hawkular-cassandra-certs/tls.peer.truststore.crt"},{"name":"TRUSTSTORE_CLIENT_AUTHORITIES","value":"/hawkular-cassandra-certs/tls.client.truststore.crt"},{"name":"TAKE_SNAPSHOT","value":"False"},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}},{"name":"MEMORY_LIMIT","valueFrom":{"resourceFieldRef":{"divisor":"0","resource":"limits.memory"}}},{"name":"CPU_LIMIT","valueFrom":{"resourceFieldRef":{"divisor":"1m","resource":"limits.cpu"}}}],"image":"registry.reg-aws.openshift.com:443/openshift3/metrics-cassandra:v3.10","imagePullPolicy":"IfNotPresent","lifecycle":{"postStart":{"exec":{"command":["/opt/apache-cassandra/bin/cassandra-poststart.sh"]}},"preStop":{"exec":{"command":["/opt/apache-cassandra/bin/cassandra-prestop.sh"]}}},"name":"hawkular-cassandra-1","ports":[{"containerPort":9042,"name":"cql-port","protocol":"TCP"},{"containerPort":9160,"name":"thrift-port","protocol":"TCP"},{"containerPort":7000,"name":"tcp-port","protocol":"TCP"},{"containerPort":7001,"name":"ssl-port","protocol":"TCP"}],"readinessProbe":{"exec":{"command":["/opt/apache-cassandra/bin/cassandra-docker-ready.sh"]},"failureThreshold":3,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"resources":{"limits":{"memory":"2G"},"requests":{"memory":"1G"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/cassandra_data","name":"cassandra-data"},{"mountPath":"/hawkular-cassandra-certs","name":"hawkular-cassandra-certs"}]}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{"supplementalGroups":[65534]},"serviceAccount":"cassandra","serviceAccountName":"cassandra","terminationGracePeriodSeconds":1800,"volumes":[{"emptyDir":{},"name":"cassandra-data"},{"name":"hawkular-cassandra-certs","secret":{"defaultMode":420,"secretName":"hawkular-cassandra-certs"}}]}}},"status":{"observedGeneration":1,"replicas":0}}
>  creationTimestamp: 2018-06-05T07:13:22Z
>  generation: 2
>  labels:
>    metrics-infra: hawkular-cassandra
>    name: hawkular-cassandra
>    type: hawkular-cassandra
>  name: hawkular-cassandra-1
>  namespace: openshift-infra
>  resourceVersion: "54406"
>  selfLink: /api/v1/namespaces/openshift-infra/replicationcontrollers/hawkular-cassandra-1
>  uid: eeb819d5-688f-11e8-a8d7-fa163ee42f0d
>spec:
>  replicas: 1
>  selector:
>    name: hawkular-cassandra-1
>  template:
>    metadata:
>      creationTimestamp: null
>      labels:
>        metrics-infra: hawkular-cassandra
>        name: hawkular-cassandra-1
>        type: hawkular-cassandra
>    spec:
>      containers:
>      - command:
>        - /opt/apache-cassandra/bin/cassandra-docker.sh
>        - --cluster_name=hawkular-metrics
>        - --data_volume=/cassandra_data
>        - --internode_encryption=all
>        - --require_node_auth=true
>        - --enable_client_encryption=true
>        - --require_client_auth=true
>        env:
>        - name: CASSANDRA_MASTER
>          value: "true"
>        - name: CASSANDRA_DATA_VOLUME
>          value: /cassandra_data
>        - name: JVM_OPTS
>          value: -Dcassandra.commitlog.ignorereplayerrors=true
>        - name: ENABLE_PROMETHEUS_ENDPOINT
>          value: "True"
>        - name: TRUSTSTORE_NODES_AUTHORITIES
>          value: /hawkular-cassandra-certs/tls.peer.truststore.crt
>        - name: TRUSTSTORE_CLIENT_AUTHORITIES
>          value: /hawkular-cassandra-certs/tls.client.truststore.crt
>        - name: TAKE_SNAPSHOT
>          value: "False"
>        - name: POD_NAMESPACE
>          valueFrom:
>            fieldRef:
>              apiVersion: v1
>              fieldPath: metadata.namespace
>        - name: MEMORY_LIMIT
>          valueFrom:
>            resourceFieldRef:
>              divisor: "0"
>              resource: limits.memory
>        - name: CPU_LIMIT
>          valueFrom:
>            resourceFieldRef:
>              divisor: 1m
>              resource: limits.cpu
>        image: registry.reg-aws.openshift.com:443/openshift3/metrics-cassandra:v3.10
>        imagePullPolicy: IfNotPresent
>        lifecycle:
>          postStart:
>            exec:
>              command:
>              - /opt/apache-cassandra/bin/cassandra-poststart.sh
>          preStop:
>            exec:
>              command:
>              - /opt/apache-cassandra/bin/cassandra-prestop.sh
>        name: hawkular-cassandra-1
>        ports:
>        - containerPort: 9042
>          name: cql-port
>          protocol: TCP
>        - containerPort: 9160
>          name: thrift-port
>          protocol: TCP
>        - containerPort: 7000
>          name: tcp-port
>          protocol: TCP
>        - containerPort: 7001
>          name: ssl-port
>          protocol: TCP
>        readinessProbe:
>          exec:
>            command:
>            - /opt/apache-cassandra/bin/cassandra-docker-ready.sh
>          failureThreshold: 3
>          periodSeconds: 10
>          successThreshold: 1
>          timeoutSeconds: 1
>        resources:
>          limits:
>            memory: 2G
>          requests:
>            memory: 1G
>        terminationMessagePath: /dev/termination-log
>        terminationMessagePolicy: File
>        volumeMounts:
>        - mountPath: /cassandra_data
>          name: cassandra-data
>        - mountPath: /hawkular-cassandra-certs
>          name: hawkular-cassandra-certs
>      dnsPolicy: ClusterFirst
>      restartPolicy: Always
>      schedulerName: default-scheduler
>      securityContext:
>        supplementalGroups:
>        - 65534
>      serviceAccount: cassandra
>      serviceAccountName: cassandra
>      terminationGracePeriodSeconds: 1800
>      volumes:
>      - emptyDir: {}
>        name: cassandra-data
>      - name: hawkular-cassandra-certs
>        secret:
>          defaultMode: 420
>          secretName: hawkular-cassandra-certs
>status:
>  availableReplicas: 1
>  fullyLabeledReplicas: 1
>  observedGeneration: 2
>  readyReplicas: 1
>  replicas: 1
>

Actions: View

Attachments on bug 1571641: 1447755 | 1453333