Bug 1533790

Description of problem:
When enable hpa with horizontal-pod-autoscaler-use-rest-clients=true, when hpa get metrics it always failed with "failed to get cpu utilization: unable to get metrics for resource cpu: failed to get pod resource metrics: the server could not find the requested resource (get services https:heapster:)"

Version-Release number of selected component (if applicable):
openshift v3.9.0-0.16.0
kubernetes v1.9.0-beta1
etcd 3.2.8

How reproducible:
Always

Steps to Reproduce:
 
1. Configure hpav2 in "/etc/origin/master/master-config.yaml" like below then restart master api & controller services

kubernetesMasterConfig:
  apiServerArguments:
    runtime-config:
    - autoscaling/v2beta1=true
  controllerArguments:
    horizontal-pod-autoscaler-use-rest-clients:
    - 'true'
    horizontal-pod-autoscaler-sync-period:
    - 10s

# systemctl restart atomic-openshift-master-api.service
# systemctl restart atomic-openshift-master-controllers.service

2. Deploy metrics-server and make sure metrics-server works well in ocp
[root@ip-172-18-11-233 deploy]# oc get svc -n kube-system
NAME             TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE
metrics-server   ClusterIP   172.30.14.24   <none>        443/TCP   2h
[root@ip-172-18-11-233 deploy]# oc get po -n kube-system
NAME                            READY     STATUS    RESTARTS   AGE
metrics-server-8946c475-7xfrv   1/1       Running   0          1h
[root@ip-172-18-11-233 deploy]# oc get --raw /apis/metrics.k8s.io/v1beta1
{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"metrics.k8s.io/v1beta1","resources":[{"name":"nodes","singularName":"","namespaced":false,"kind":"NodeMetrics","verbs":["get","list"]},{"name":"pods","singularName":"","namespaced":true,"kind":"PodMetrics","verbs":["get","list"]}]}


3. Create a rc and hpa v2beta1 resource then check the hpa status
[root@ip-172-18-11-233 deploy]# oc adm new-project dma1
Created project dma1
[root@ip-172-18-11-233 deploy]# oc create -f https://raw.githubusercontent.com/mdshuai/testfile-openshift/master/k8s/autoscaling/hpa/rc.yaml -n dma1
replicationcontroller "hello-openshift" created
[root@ip-172-18-11-233 deploy]# oc create -f https://raw.githubusercontent.com/mdshuai/testfile-openshift/master/k8s/autoscaling/hpa/resource.yaml -n dma1
horizontalpodautoscaler "resource-hpa" created
[root@ip-172-18-11-233 deploy]# 
[root@ip-172-18-11-233 deploy]# oc get hpa.v2beta1.autoscaling -n dma1
NAME           REFERENCE                               TARGETS           MINPODS   MAXPODS   REPLICAS   AGE
resource-hpa   ReplicationController/hello-openshift   <unknown> / 80%   2         10        1          16s
[root@ip-172-18-11-233 deploy]# oc describe hpa.v2beta1.autoscaling resource-hpa -n dma1
Name:                                                  resource-hpa
Namespace:                                             dma1
Labels:                                                <none>
Annotations:                                           <none>
CreationTimestamp:                                     Fri, 12 Jan 2018 03:10:46 -0500
Reference:                                             ReplicationController/hello-openshift
Metrics:                                               ( current / target )
  resource cpu on pods  (as a percentage of request):  <unknown> / 80%
Min replicas:                                          2
Max replicas:                                          10
ReplicationController pods:                            2 current / 2 desired
Conditions:
  Type           Status  Reason                   Message
  ----           ------  ------                   -------
  AbleToScale    True    SucceededGetScale        the HPA controller was able to get the target's current scale
  ScalingActive  False   FailedGetResourceMetric  the HPA was unable to compute the replica count: unable to get metrics for resource cpu: failed to get pod resource metrics: the server could not find the requested resource (get services https:heapster:)
Events:
  Type     Reason                        Age               From                       Message
  ----     ------                        ----              ----                       -------
  Normal   SuccessfulRescale             25s               horizontal-pod-autoscaler  New size: 2; reason: Current number of replicas below Spec.MinReplicas
  Warning  FailedGetResourceMetric       5s (x2 over 15s)  horizontal-pod-autoscaler  unable to get metrics for resource cpu: failed to get pod resource metrics: the server could not find the requested resource (get services https:heapster:)
  Warning  FailedComputeMetricsReplicas  5s (x2 over 15s)  horizontal-pod-autoscaler  failed to get cpu utilization: unable to get metrics for resource cpu: failed to get pod resource metrics: the server could not find the requested resource (get services https:heapster:)

Actual results:
3. horizontal-pod-autoscaler  failed to get cpu utilization: unable to get metrics for resource cpu: failed to get pod resource metrics: the server could not find the requested resource (get services https:heapster:)

Expected results:
3. hpa should get metrics success

Additional info: