Kubernetes Cloud Autoscalar in Terraform - Part 1
Here's an example of a Kubernetes Cluster that creates the Cloud Autoscaler deployment using Terraform dynamic Modules instead of using a YAML file.
Join the DZone community and get the full member experience.
Join For FreeCluster Autoscaler - It is a component that automatically adjusts the size of a Kubernetes Cluster so that all pods have a place to run and there are no unneeded nodes. It can be created by using the YAML file, Helm Chart, or Terraform. The following example creates the Cloud Autoscalar deployment using Terraform dynamic Modules.
Corresponding to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml
Resource kubernetes_deployment:-
xxxxxxxxxx
cluster-autoscaler.tf:-
resource "kubernetes_deployment" "cluster-autoscaler" {
metadata {
name = var.cluster-autoscaler-data["cluster-autoscaler-name"]
namespace = var.cluster-autoscaler-data["namespace"]
labels = {
"app" = var.cluster-autoscaler-data["cluster-autoscaler-label"]
}
}
spec {
replicas = 1
selector {
match_labels = {
"app" = var.cluster-autoscaler-data["cluster-autoscaler-label"]
}
}
template {
metadata {
labels = {
"app" = var.cluster-autoscaler-data["cluster-autoscaler-label"]
}
annotations = {
"prometheus.io/port" = "8085"
"prometheus.io/scrape" = "true"
}
}
spec {
automount_service_account_token = true
termination_grace_period_seconds = 300
service_account_name = var.service-account-name
container {
image = var.cluster-autoscaler-data["image_name"]
name = "cluster-autoscaler"
command = ["./cluster-autoscaler",
"--v=4",
" --stderrthreshold=info",
"--cloud-provider=aws",
"--skip-nodes-with-local-storage=false",
"--expander=least-waste",
"--node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/${var.eks_cluster_name}"
]
resources {
limits {
cpu = "100m"
memory = "300Mi"
}
requests {
cpu = "100m"
memory = "300Mi"
}
}
volume_mount {
name = "ssl-certs"
mount_path = "/etc/ssl/certs/ca-certificates.crt"
read_only = "true"
}
}
volume {
name = "ssl-certs"
host_path {
path = "/etc/ssl/certs/ca-bundle.crt"
}
}
}
}
}
}
variable.tf :-
variable "cluster-autoscaler-data" {}
variable "service-account-name" {}
variable "eks_cluster_name" {}
Service Account Module:-
xxxxxxxxxx
service-account.tf:-
resource "kubernetes_service_account" "service-account" {
metadata {
name = lower(var.service_account_name)
namespace = var.namespace
labels = {
"${var.label1}" = var.label-value1
}
}
}
data "kubernetes_secret" "service-account-secret" {
metadata {
name = kubernetes_service_account.service-account.default_secret_name
namespace = var.namespace
}
}
variable.tf:-
variable "service_account_name" {}
variable "namespace" {}
variable "label-value1" {}
variable "label1" {
default = "name"
}
Cluster Role Binding Module:-
xxxxxxxxxx
clusterrolebinding-resource.tf:-
resource "kubernetes_cluster_role_binding" "clusterrolebinding-res" {
metadata {
name = lower(var.cluster-role-binding-name)
labels = {
"${var.label1}" = var.label-value1
}
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "ClusterRole"
name = var.cluster-role
}
subject {
kind = "ServiceAccount"
name = var.service-account-name
namespace = var.namespace
}
}
variable.tf:-
variable "cluster-role-binding-name" {}
variable "cluster-role" {}
variable "service-account-name" {}
variable "namespace" {}
variable "label-value1" {}
variable "label1" {
default = "name"
}
Role Binding Module in Terraform:-
xxxxxxxxxx
rolebinding-res.tf File:-
resource "kubernetes_role_binding" "rolebinding-res" {
metadata {
name = lower(var.role-binding-name)
namespace = var.namespace
labels = {
"${var.label1}" = var.label-value1
}
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = var.role-name
}
subject {
kind = "ServiceAccount"
name = var.service-account-name
namespace = var.namespace
}
}
variable.tf :-
variable "role-binding-name" {}
variable "role-name" {}
variable "service-account-name" {}
variable "namespace" {}
variable "label-value1" {}
variable "label1" {
default = "name"
}
Role Module:-
xxxxxxxxxx
k8role.tf:-
resource "kubernetes_role" "k8role" {
metadata {
name = lower(var.role-name)
namespace = var.namespace
labels = {
"${var.label1}" = var.label-value1
}
}
dynamic "rule" {
for_each = var.rules
content {
api_groups = rule.value.api_groups
resources = rule.value.resources
verbs = rule.value.verbs
}
}
}
variable.tf File:-
variable "role-name" {}
variable "namespace" {}
variable "rules" {}
variable "label-value1" {}
variable "label1" {
default = "name"
}
Cluster Role Module:-
xxxxxxxxxx
clusterrole.tf:-
resource "kubernetes_cluster_role" "clusterrole" {
metadata {
name = lower(var.cluster-role-name)
labels = {
"${var.label1}" = var.label-value1
}
}
dynamic "rule" {
for_each = var.rules
content {
api_groups = rule.value.api_groups
resources = rule.value.resources
verbs = rule.value.verbs
}
}
}
variable.tf :-
variable "cluster-role-name" {}
variable "rules" {}
variable "label-value1" {}
variable "label1" {
default = "name"
}
Calling above-defined Modules:-
xxxxxxxxxx
module "cluster-autoscaler-sa" {
source = "./../service-account"
service_account_name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-sa"
label1 = var.cluster-autoscaler-data["label1"]
label-value1 = var.cluster-autoscaler-data["label-value1"]
namespace = var.cluster-autoscaler-data["namespace"]
}
module "autoscaler-deployment" {
source = "./../cluster-autoscaler-deployment"
cluster-autoscaler-data = var.cluster-autoscaler-data
service-account-name = module.cluster-autoscaler-sa.service-account-name
eks_cluster_name = var.eks_cluster_name
}
module "cluster-autoscaler-cluster-role" {
source = "./../clusterrole"
cluster-role-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-clusterrole"
label1 = var.cluster-autoscaler-data["label1"]
label-value1 = var.cluster-autoscaler-data["label-value1"]
rules = [
{
api_groups = [""]
resources = ["events", "endpoints"]
verbs = ["create", "patch"]
},
{
api_groups = [""]
resources = ["pods/eviction"]
verbs = ["create"]
},
{
api_groups = [""]
resources = ["pods/status"]
verbs = ["update"]
},
{
api_groups = [""]
resources = ["endpoints"]
resource_names = ["cluster-autoscaler"]
verbs = ["get", "update"]
},
{
api_groups = [""]
resources = ["nodes"]
verbs = ["watch", "list", "get", "update"]
},
{
api_groups = [""]
resources = ["pods", "services", "replicationcontrollers", "persistentvolumeclaims", "persistentvolumes"]
verbs = ["watch", "list", "get"]
}, {
api_groups = ["extensions"]
resources = ["replicasets", "daemonsets"]
verbs = ["watch", "list", "get"]
}
, {
api_groups = ["policy"]
resources = ["poddisruptionbudgets"]
verbs = ["watch", "list"]
}
, {
api_groups = ["apps"]
resources = ["statefulsets", "replicasets", "daemonsets"]
verbs = ["watch", "list", "get"]
}
, {
api_groups = ["storage.k8s.io"]
resources = ["storageclasses", "csinodes"]
verbs = ["watch", "list", "get"]
}
, {
api_groups = ["batch", "extensions"]
resources = ["jobs"]
verbs = ["get", "list", "watch", "patch"]
}
, {
api_groups = ["coordination.k8s.io"]
resources = ["leases"]
verbs = ["create"]
},
{
api_groups = ["coordination.k8s.io"]
resource_names = ["cluster-autoscaler"]
resources = ["leases"]
verbs = ["get", "update"]
}
]
}
module "cluster-autoscaler-clusterrolebinding" {
source = "./../clusterrolebinding"
cluster-role-binding-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-clusterrolebinding"
cluster-role = module.cluster-autoscaler-cluster-role.cluster-role-name
service-account-name = module.cluster-autoscaler-sa.service-account-name
label1 = var.cluster-autoscaler-data["label1"]
label-value1 = var.cluster-autoscaler-data["label-value1"]
namespace = var.cluster-autoscaler-data["namespace"]
}
module "autoscaler-role" {
source = "./../role"
role-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-role"
label1 = var.cluster-autoscaler-data["label1"]
label-value1 = var.cluster-autoscaler-data["label-value1"]
namespace = var.cluster-autoscaler-data["namespace"]
rules = [
{
api_groups = [""]
resources = ["configmaps"]
verbs = ["create", "list", "watch"]
},
{
api_groups = [""]
resources = ["configmaps"]
resource_names = ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
verbs = ["delete", "get", "update", "watch"]
}
]
}
module "autoscaler-rolebinding" {
source = "./../rolebinding"
role-binding-name = "${var.cluster-autoscaler-data["cluster-autoscaler-name"]}-rolebinding"
role-name = module.autoscaler-role.role-name
service-account-name = module.cluster-autoscaler-sa.service-account-name
label1 = var.cluster-autoscaler-data["label1"]
label-value1 = var.cluster-autoscaler-data["label-value1"]
namespace = var.cluster-autoscaler-data["namespace"]
}
Data passed in parameters:-
xxxxxxxxxx
cluster-autoscaler-data = {
cluster-autoscaler-name = "k8demo-cluster-autoscaler"
cluster-autoscaler-label = "k8demo-cluster-autoscaler"
label1 = "k8s-addon"
label-value1 = "cluster-autoscaler.addons.k8s.io"
namespace = "kube-system"
image_name = "k8s.gcr.io/cluster-autoscaler:v1.14.7"
}
service-account-name = "k8demo-service-account"
eks_cluster_name = "k8demo-Cluster"
Stay tuned for the second article.
Reference Article:- https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md
Opinions expressed by DZone contributors are their own.
Comments