300 lines
13 KiB
Text
300 lines
13 KiB
Text
|
{% set daemon_args = "$DAEMON_ARGS" -%}
|
||
|
|
{% if grains['os_family'] == 'RedHat' -%}
|
||
|
|
{% set daemon_args = "" -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set cloud_provider = "" -%}
|
||
|
|
{% set cloud_config = "" -%}
|
||
|
|
{% set cloud_config_mount = "" -%}
|
||
|
|
{% set cloud_config_volume = "" -%}
|
||
|
|
{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%}
|
||
|
|
{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%}
|
||
|
|
|
||
|
|
{% set srv_kube_path = "/srv/kubernetes" -%}
|
||
|
|
{% set srv_sshproxy_path = "/srv/sshproxy" -%}
|
||
|
|
|
||
|
|
{% if grains.cloud is defined -%}
|
||
|
|
{% if grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%}
|
||
|
|
{% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%}
|
||
|
|
{% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%}
|
||
|
|
{% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
|
||
|
|
{% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
|
||
|
|
{% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% if grains.cloud in ['openstack'] -%}
|
||
|
|
{% set cloud_config_mount = "{\"name\": \"instanceid\",\"mountPath\": \"/var/lib/cloud/data/instance-id\",\"readOnly\": true}," -%}
|
||
|
|
{% set cloud_config_volume = "{\"name\": \"instanceid\",\"hostPath\": {\"path\": \"/var/lib/cloud/data/instance-id\"}}," -%}
|
||
|
|
{% endif -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set advertise_address = "" -%}
|
||
|
|
{% if grains.advertise_address is defined -%}
|
||
|
|
{% set advertise_address = "--advertise-address=" + grains.advertise_address -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set proxy_ssh_options = "" -%}
|
||
|
|
{% if grains.proxy_ssh_user is defined -%}
|
||
|
|
{% set proxy_ssh_options = "--ssh-user=" + grains.proxy_ssh_user + " --ssh-keyfile=/srv/sshproxy/.sshkeyfile" -%}
|
||
|
|
{# Append 40 characters onto command to work around #9822. #}
|
||
|
|
{# If mount list changes, this may also need to change. #}
|
||
|
|
{% set proxy_ssh_options = proxy_ssh_options + " " -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set address = "--address=127.0.0.1" -%}
|
||
|
|
|
||
|
|
{% set bind_address = "" -%}
|
||
|
|
{% if grains.publicAddressOverride is defined -%}
|
||
|
|
{% set bind_address = "--bind-address=" + grains.publicAddressOverride -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set storage_backend = "" -%}
|
||
|
|
{% if pillar['storage_backend'] is defined -%}
|
||
|
|
{% set storage_backend = "--storage-backend=" + pillar['storage_backend'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
{% set etcd_servers = "--etcd-servers=http://127.0.0.1:2379" -%}
|
||
|
|
{% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%}
|
||
|
|
|
||
|
|
{% set max_requests_inflight = "" -%}
|
||
|
|
{% set target_ram_mb = "" -%}
|
||
|
|
{% if pillar['num_nodes'] is defined -%}
|
||
|
|
# If the cluster is large, increase max-requests-inflight limit in apiserver.
|
||
|
|
{% if pillar['num_nodes']|int >= 1000 -%}
|
||
|
|
{% set max_requests_inflight = "--max-requests-inflight=1500 --max-mutating-requests-inflight=500" -%}
|
||
|
|
{% endif -%}
|
||
|
|
# Set amount of memory available for apiserver based on number of nodes.
|
||
|
|
# TODO: Once we start setting proper requests and limits for apiserver
|
||
|
|
# we should reuse the same logic here instead of current heuristic.
|
||
|
|
{% set tmp_ram_mb = pillar['num_nodes']|int * 60 %}
|
||
|
|
{% set target_ram_mb = "--target-ram-mb=" + tmp_ram_mb|string -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set service_cluster_ip_range = "" -%}
|
||
|
|
{% if pillar['service_cluster_ip_range'] is defined -%}
|
||
|
|
{% set service_cluster_ip_range = "--service-cluster-ip-range=" + pillar['service_cluster_ip_range'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set cert_file = "--tls-cert-file=/srv/kubernetes/server.cert" -%}
|
||
|
|
{% set key_file = "--tls-private-key-file=/srv/kubernetes/server.key" -%}
|
||
|
|
{% set kubelet_cert_file = "--kubelet-client-certificate=/srv/kubernetes/kubeapiserver.cert" -%}
|
||
|
|
{% set kubelet_key_file = "--kubelet-client-key=/srv/kubernetes/kubeapiserver.key" -%}
|
||
|
|
{% set client_ca_file = "" -%}
|
||
|
|
|
||
|
|
{% set secure_port = "6443" -%}
|
||
|
|
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'photon-controller', 'openstack'] %}
|
||
|
|
{% set secure_port = "443" -%}
|
||
|
|
{% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set min_request_timeout = "" -%}
|
||
|
|
{% if grains.minRequestTimeout is defined -%}
|
||
|
|
{% set min_request_timeout = "--min-request-timeout=" + grains.minRequestTimeout -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set token_auth_file = " --token-auth-file=/dev/null" -%}
|
||
|
|
{% set basic_auth_file = "" -%}
|
||
|
|
{% set authz_mode = "" -%}
|
||
|
|
{% set abac_policy_file = "" -%}
|
||
|
|
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'photon-controller', 'openstack'] %}
|
||
|
|
{% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
|
||
|
|
{% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
|
||
|
|
{% set authz_mode = " --authorization-mode=ABAC" -%}
|
||
|
|
{% set abac_policy_file = " --authorization-policy-file=/srv/kubernetes/abac-authz-policy.jsonl" -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set webhook_authentication_config = "" -%}
|
||
|
|
{% set webhook_authn_config_mount = "" -%}
|
||
|
|
{% set webhook_authn_config_volume = "" -%}
|
||
|
|
{% if grains.webhook_authentication_config is defined -%}
|
||
|
|
{% set webhook_authentication_config = " --authentication-token-webhook-config-file=" + grains.webhook_authentication_config -%}
|
||
|
|
{% set webhook_authn_config_mount = "{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"" + grains.webhook_authentication_config + "\", \"readOnly\": false}," -%}
|
||
|
|
{% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\"}}," -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set webhook_authorization_config = "" -%}
|
||
|
|
{% set webhook_config_mount = "" -%}
|
||
|
|
{% set webhook_config_volume = "" -%}
|
||
|
|
{% if grains.webhook_authorization_config is defined -%}
|
||
|
|
{% set webhook_authorization_config = " --authorization-webhook-config-file=" + grains.webhook_authorization_config -%}
|
||
|
|
{% set webhook_config_mount = "{\"name\": \"webhookconfigmount\",\"mountPath\": \"" + grains.webhook_authorization_config + "\", \"readOnly\": false}," -%}
|
||
|
|
{% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\"}}," -%}
|
||
|
|
{% set authz_mode = authz_mode + ",Webhook" -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set image_review_config = "" -%}
|
||
|
|
{% set admission_controller_config_mount = "" -%}
|
||
|
|
{% set admission_controller_config_volume = "" -%}
|
||
|
|
{% set image_policy_webhook_config_mount = "" -%}
|
||
|
|
{% set image_policy_webhook_config_volume = "" -%}
|
||
|
|
{% if grains.image_review_config is defined -%}
|
||
|
|
{% set image_review_config = " --admission-control-config-file=" + grains.image_review_config -%}
|
||
|
|
{% set admission_controller_config_mount = "{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"" + grains.image_review_config + "\", \"readOnly\": false}," -%}
|
||
|
|
{% set admission_controller_config_volume = "{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"" + grains.image_review_config + "\"}}," -%}
|
||
|
|
{% set image_policy_webhook_config_mount = "{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," -%}
|
||
|
|
{% set image_policy_webhook_config_volume = "{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set admission_control = "" -%}
|
||
|
|
{% if pillar['admission_control'] is defined -%}
|
||
|
|
{% set admission_control = "--admission-control=" + pillar['admission_control'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set runtime_config = "" -%}
|
||
|
|
{% if grains.runtime_config is defined -%}
|
||
|
|
{% set runtime_config = "--runtime-config=" + grains.runtime_config -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set feature_gates = "" -%}
|
||
|
|
{% if grains.feature_gates is defined -%}
|
||
|
|
{% set feature_gates = "--feature-gates=" + grains.feature_gates -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set log_level = pillar['log_level'] -%}
|
||
|
|
{% if pillar['api_server_test_log_level'] is defined -%}
|
||
|
|
{% set log_level = pillar['api_server_test_log_level'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set enable_garbage_collector = "" -%}
|
||
|
|
{% if pillar['enable_garbage_collector'] is defined -%}
|
||
|
|
{% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set etcd_quorum_read = "" %}
|
||
|
|
{% if pillar['etcd_quorum_read'] is defined -%}
|
||
|
|
{% set etcd_quorum_read = "--etcd_quorum_read=" + pillar['etcd_quorum_read'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read -%}
|
||
|
|
{% set params = params + " " + cert_file + " " + key_file + " " + kubelet_cert_file + " " + kubelet_key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config + image_review_config -%}
|
||
|
|
|
||
|
|
# test_args has to be kept at the end, so they'll overwrite any prior configuration
|
||
|
|
{% if pillar['apiserver_test_args'] is defined -%}
|
||
|
|
{% set params = params + " " + pillar['apiserver_test_args'] -%}
|
||
|
|
{% endif -%}
|
||
|
|
|
||
|
|
{
|
||
|
|
"apiVersion": "v1",
|
||
|
|
"kind": "Pod",
|
||
|
|
"metadata": {
|
||
|
|
"name":"kube-apiserver",
|
||
|
|
"namespace": "kube-system",
|
||
|
|
"labels": {
|
||
|
|
"tier": "control-plane",
|
||
|
|
"component": "kube-apiserver"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"spec":{
|
||
|
|
"hostNetwork": true,
|
||
|
|
"containers":[
|
||
|
|
{
|
||
|
|
"name": "kube-apiserver",
|
||
|
|
"image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
|
||
|
|
"resources": {
|
||
|
|
"requests": {
|
||
|
|
"cpu": "250m"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"command": [
|
||
|
|
"/bin/sh",
|
||
|
|
"-c",
|
||
|
|
"/usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
|
||
|
|
],
|
||
|
|
"livenessProbe": {
|
||
|
|
"httpGet": {
|
||
|
|
"host": "127.0.0.1",
|
||
|
|
"port": 8080,
|
||
|
|
"path": "/healthz"
|
||
|
|
},
|
||
|
|
"initialDelaySeconds": 15,
|
||
|
|
"timeoutSeconds": 15
|
||
|
|
},
|
||
|
|
"ports":[
|
||
|
|
{ "name": "https",
|
||
|
|
"containerPort": {{secure_port}},
|
||
|
|
"hostPort": {{secure_port}}},{
|
||
|
|
"name": "local",
|
||
|
|
"containerPort": 8080,
|
||
|
|
"hostPort": 8080}
|
||
|
|
],
|
||
|
|
"volumeMounts": [
|
||
|
|
{{cloud_config_mount}}
|
||
|
|
{{additional_cloud_config_mount}}
|
||
|
|
{{webhook_config_mount}}
|
||
|
|
{{webhook_authn_config_mount}}
|
||
|
|
{{admission_controller_config_mount}}
|
||
|
|
{{image_policy_webhook_config_mount}}
|
||
|
|
{ "name": "srvkube",
|
||
|
|
"mountPath": "{{srv_kube_path}}",
|
||
|
|
"readOnly": true},
|
||
|
|
{ "name": "logfile",
|
||
|
|
"mountPath": "/var/log/kube-apiserver.log",
|
||
|
|
"readOnly": false},
|
||
|
|
{ "name": "etcssl",
|
||
|
|
"mountPath": "/etc/ssl",
|
||
|
|
"readOnly": true},
|
||
|
|
{ "name": "usrsharecacerts",
|
||
|
|
"mountPath": "/usr/share/ca-certificates",
|
||
|
|
"readOnly": true},
|
||
|
|
{ "name": "varssl",
|
||
|
|
"mountPath": "/var/ssl",
|
||
|
|
"readOnly": true},
|
||
|
|
{ "name": "etcopenssl",
|
||
|
|
"mountPath": "/etc/openssl",
|
||
|
|
"readOnly": true},
|
||
|
|
{ "name": "etcpki",
|
||
|
|
"mountPath": "/etc/pki",
|
||
|
|
"readOnly": true},
|
||
|
|
{ "name": "srvsshproxy",
|
||
|
|
"mountPath": "{{srv_sshproxy_path}}",
|
||
|
|
"readOnly": false}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"volumes":[
|
||
|
|
{{cloud_config_volume}}
|
||
|
|
{{additional_cloud_config_volume}}
|
||
|
|
{{webhook_config_volume}}
|
||
|
|
{{webhook_authn_config_volume}}
|
||
|
|
{{admission_controller_config_volume}}
|
||
|
|
{{image_policy_webhook_config_volume}}
|
||
|
|
{ "name": "srvkube",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "{{srv_kube_path}}"}
|
||
|
|
},
|
||
|
|
{ "name": "logfile",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "/var/log/kube-apiserver.log"}
|
||
|
|
},
|
||
|
|
{ "name": "etcssl",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "/etc/ssl"}
|
||
|
|
},
|
||
|
|
{ "name": "usrsharecacerts",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "/usr/share/ca-certificates"}
|
||
|
|
},
|
||
|
|
{ "name": "varssl",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "/var/ssl"}
|
||
|
|
},
|
||
|
|
{ "name": "etcopenssl",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "/etc/openssl"}
|
||
|
|
},
|
||
|
|
{ "name": "etcpki",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "/etc/pki"}
|
||
|
|
},
|
||
|
|
{ "name": "srvsshproxy",
|
||
|
|
"hostPath": {
|
||
|
|
"path": "{{srv_sshproxy_path}}"}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}}
|