[NCP] Terraform으로 Bastion + NKS 배포하기

아키텍처

• NCP 환경
  • Bastion VM : 1EA
  • NKS Cluster : Control Plane 1EA, NodePool 1EA
  • Nat Gateway : 1EA

## 디렉토리 tree
├── bastion.tf
├── data.tf
├── nks.tf
├── output.tf
├── security.tf
├── terraform.tfvars
├── user_data.sh
├── variables.tf
├── version.tf
└── vpc.tf

## terraform.tfvars
vpc_name = "my-vpc"
vpc_cidr = "10.10.0.0/16"
subnet = [ "10.10.1.0/24", "10.10.10.0/24", "10.10.30.0/24" , "10.10.35.0/24", "10.10.50.0/24" ]
subnet_type = [ "PUBLIC", "PRIVATE" ]
usage_type = [ "GEN", "LOADB", "NATGW"]
zone = "KR-2"
region = "KR"
site = "pub"
access_key = "{본인 계정의 key 값 입력}"
secret_key = "{본인 계정의 key 값 입력}"
bastion_init_sh = "./user_data.sh"

 

- vpc.tf

provider "ncloud" {
  access_key  = var.access_key
  secret_key  = var.secret_key
  region      = var.region
  site        = var.site
  support_vpc = var.support_vpc
}

## VPC
resource "ncloud_vpc" "vpc" {
  name            = "${var.vpc_name}-terraform"
  ipv4_cidr_block = var.vpc_cidr
}


## Subnet
resource "ncloud_subnet" "pub_subnet" {
  vpc_no         = ncloud_vpc.vpc.vpc_no
  subnet         = var.subnet[0]
  zone           = var.zone
  network_acl_no = ncloud_network_acl.network_acl_public.network_acl_no
  subnet_type    = var.subnet_type[0]
  name           = "${var.vpc_name}-pub-subnet1"
  usage_type     = var.usage_type[0]
}

resource "ncloud_subnet" "priv_subnet" {
  vpc_no         = ncloud_vpc.vpc.vpc_no
  subnet         = var.subnet[1]
  zone           = var.zone
  network_acl_no = ncloud_network_acl.network_acl_private.network_acl_no
  subnet_type    = var.subnet_type[1]
  name           = "${var.vpc_name}-priv-subnet1"
  usage_type     = var.usage_type[0]
}

resource "ncloud_subnet" "lb_pub_subnet" {
  vpc_no         = ncloud_vpc.vpc.vpc_no
  subnet         = var.subnet[2]
  zone           = var.zone
  network_acl_no = ncloud_network_acl.network_acl_public.network_acl_no
  subnet_type    = var.subnet_type[0]
  name           = "${var.vpc_name}-lb-pub-subnet1"
  usage_type     = var.usage_type[1]
}

resource "ncloud_subnet" "lb_priv_subnet" {
  vpc_no         = ncloud_vpc.vpc.vpc_no
  subnet         = var.subnet[3]
  zone           = var.zone
  network_acl_no = ncloud_network_acl.network_acl_private.network_acl_no
  subnet_type    = var.subnet_type[1]
  name           = "${var.vpc_name}-lb-priv-subnet1"
  usage_type     = var.usage_type[1]
}

resource "ncloud_subnet" "ngw_subnet" {
  vpc_no         = ncloud_vpc.vpc.vpc_no
  subnet         = var.subnet[4]
  zone           = var.zone
  network_acl_no = ncloud_network_acl.network_acl_public.network_acl_no
  subnet_type    = var.subnet_type[0]
  name           = "${var.vpc_name}-ngw-subnet1"
  usage_type     = var.usage_type[2]
}

##ACL
resource "ncloud_network_acl" "network_acl_public" {
  vpc_no = ncloud_vpc.vpc.vpc_no
  name   = "${var.vpc_name}-public-acl"
}

resource "ncloud_network_acl" "network_acl_private" {
  vpc_no = ncloud_vpc.vpc.vpc_no
  name   = "${var.vpc_name}-private-acl"
}

## Nat gateway
resource "ncloud_nat_gateway" "vpc_ngw" {
  vpc_no    = ncloud_vpc.vpc.vpc_no
  subnet_no = ncloud_subnet.ngw_subnet.subnet_no
  zone      = var.zone
  name      = "${var.vpc_name}-k8s-nat-gw"
}

## route table
resource "ncloud_route_table" "my_cluster_rt" {
  vpc_no = ncloud_vpc.vpc.vpc_no
  #route_table_no        = ncloud_route_table.my_cluster_rt.route_table_no
  supported_subnet_type = var.subnet_type[1]
  name                  = "${var.vpc_name}-k8s-rt"
}

resource "ncloud_route" "my_cluster_route" {
  route_table_no         = ncloud_route_table.my_cluster_rt.route_table_no
  destination_cidr_block = "0.0.0.0/0"
  target_type            = var.usage_type[2]
  target_name            = ncloud_nat_gateway.vpc_ngw.name
  target_no              = ncloud_nat_gateway.vpc_ngw.nat_gateway_no
}

resource "ncloud_route_table_association" "my_k8s_cluster_table_subnet" {
  route_table_no = ncloud_route_table.my_cluster_rt.route_table_no
  subnet_no      = ncloud_subnet.priv_subnet.subnet_no
}

## netowrk interface
resource "ncloud_network_interface" "bastion-interfacae" {
  name                  = "bastion-interface"
  subnet_no             = ncloud_subnet.pub_subnet.subnet_no
  access_control_groups = [ncloud_access_control_group.bastion-acg.id]
}

resource "ncloud_network_interface" "cluster-interface" {
  name                  = "node-pool-interface"
  subnet_no             = ncloud_subnet.priv_subnet.subnet_no
  access_control_groups = [ncloud_access_control_group.cluster_acg.id]
}

 

- bastion.tf

resource "ncloud_login_key" "loginkey" {
  key_name = "my-terraform-1-key"
}

# 키 파일 로컬 저장
resource "local_file" "key_file" {
  filename        = "${path.module}/my-key.pem" # 키 파일이 저장될 경로
  content         = ncloud_login_key.loginkey.private_key
  file_permission = "0600" # 파일 권한 설정
}

resource "ncloud_server" "bastion_server" {
  subnet_no           = ncloud_subnet.pub_subnet.subnet_no
  name                = "${var.vpc_name}-bastion"
  login_key_name      = ncloud_login_key.loginkey.key_name
  server_image_number = data.ncloud_server_image_numbers.server_images.image_number_list.0.server_image_number
  server_spec_code    = data.ncloud_server_specs.spec.server_spec_list.0.server_spec_code

  # init_script_no = ncloud_init_script.init_script.id
  depends_on = [ncloud_nks_cluster.my_cluster]
  network_interface {
    network_interface_no = ncloud_network_interface.bastion-interfacae.id
    order                = 0
  }
}

resource "null_resource" "bastion_setup" {
  depends_on = [
    ncloud_server.bastion_server,
    data.ncloud_root_password.default
  ]

  # 먼저 스크립트 파일 전송
  provisioner "file" {
    source      = var.bastion_init_sh
    destination = "/root/user_data.sh"

    connection {
      type     = "ssh"
      user     = "root"
      password = data.ncloud_root_password.default.root_password
      host     = ncloud_public_ip.bastion_pub_ip.public_ip
      timeout  = "5m"
    }
  }
  provisioner "remote-exec" {
    inline = [
      "chmod +x /root/user_data.sh",
      "export cluster_name='${ncloud_nks_cluster.my_cluster.name}'",
      "export cluster_uuid='${ncloud_nks_cluster.my_cluster.uuid}'",
      "export NCP_ACCESS_KEY='${var.access_key}'",
      "export NCP_SECRET_KEY='${var.secret_key}'",
      "/root/user_data.sh"
    ]

    connection {
      type     = "ssh"
      user     = "root"
      password = data.ncloud_root_password.default.root_password
      host     = ncloud_public_ip.bastion_pub_ip.public_ip
      timeout  = "5m"
    }
  }
}
resource "ncloud_public_ip" "bastion_pub_ip" {
  server_instance_no = ncloud_server.bastion_server.id
}

 

- nks.tf

# NKS create
resource "ncloud_nks_cluster" "my_cluster" {
  hypervisor_code      = "KVM"
  cluster_type         = "SVR.VNKS.STAND.C002.M008.G003"
  login_key_name       = ncloud_login_key.loginkey.key_name
  name                 = "${var.vpc_name}-cluster"
  lb_private_subnet_no = ncloud_subnet.lb_priv_subnet.subnet_no
  lb_public_subnet_no  = ncloud_subnet.lb_pub_subnet.subnet_no
  subnet_no_list       = [ncloud_subnet.priv_subnet.subnet_no]
  kube_network_plugin  = "cilium"
  vpc_no               = ncloud_vpc.vpc.vpc_no
  zone                 = var.zone
}

# node pool create
resource "ncloud_nks_node_pool" "node_pool" {
  cluster_uuid     = ncloud_nks_cluster.my_cluster.uuid
  node_pool_name   = "${var.vpc_name}-nod-pool-1"
  node_count       = 2
  software_code    = data.ncloud_nks_server_images.image.images[0].value
  server_spec_code = data.ncloud_nks_server_products.product.products.0.value
  storage_size     = 100
  autoscale {
    enabled = false
    min     = 2
    max     = 2
  }
}

 

- security.tf

## ACG
resource "ncloud_access_control_group" "bastion-acg" {
  name   = "bastion-acg"
  vpc_no = ncloud_vpc.vpc.vpc_no
}

resource "ncloud_access_control_group_rule" "bastion_acg_rule" {
  access_control_group_no = ncloud_access_control_group.bastion-acg.id
  inbound {
    protocol    = "TCP"
    ip_block    = "${data.http.local_public_ip.response_body}/32"
    port_range  = "22"
    description = "local to vm ssh"
  }

  inbound {
    protocol    = "TCP"
    ip_block    = ""
    port_range  = "1-65535"
    description = "local"
  }

  outbound {
    protocol    = "TCP"
    ip_block    = "0.0.0.0/0"
    port_range  = "1-65535"
    description = "outbound"
  }
}

resource "ncloud_access_control_group" "cluster_acg" {
  name   = "cluster-acg"
  vpc_no = ncloud_vpc.vpc.vpc_no
}

resource "ncloud_access_control_group_rule" "cluster_acg_rule" {
  access_control_group_no = ncloud_access_control_group.cluster_acg.id
  inbound {
    protocol   = "TCP"
    ip_block   = "0.0.0.0/0"
    port_range = "1-65535"
  }
  outbound {
    protocol    = "TCP"
    ip_block    = "0.0.0.0/0"
    port_range  = "1-65535"
    description = "outbound"
  }
}

 

- data.tf

data "ncloud_server_image_numbers" "server_images" {
  server_image_name = "ubuntu-22.04-base"
  filter {
    name   = "hypervisor_type"
    values = ["KVM"]
  }
}

data "ncloud_server_specs" "spec" {
  filter {
    name   = "server_spec_code"
    values = ["c2-g3"]
  }
}

data "http" "local_public_ip" {
  url = "https://api.ipify.org?format=text"
}

data "ncloud_root_password" "default" {
  server_instance_no = ncloud_server.bastion_server.id
  private_key        = ncloud_login_key.loginkey.private_key
}

data "ncloud_nks_server_images" "image" {
  hypervisor_code = "KVM"
  filter {
    name   = "label"
    values = ["ubuntu-22.04"]
    regex  = true
  }
}

data "ncloud_nks_server_products" "product" {
  software_code = data.ncloud_nks_server_images.image.images[0].value
  zone          = "KR-2"

  filter {
    name   = "product_type"
    values = ["STAND"]
  }

  filter {
    name   = "cpu_count"
    values = ["2"]
  }

  filter {
    name   = "memory_size"
    values = ["8GB"]
  }
}

 

- output.tf

# 출력 설정
output "private_key" {
  value     = ncloud_login_key.loginkey.private_key
  sensitive = true
}

output "root_password" {
  value     = data.ncloud_root_password.default
  sensitive = true
}

output "public_ip" {
  value = ncloud_public_ip.bastion_pub_ip.public_ip
}

 

- variables.tf

variable "vpc_name" {}
variable "vpc_cidr" {}
variable "zone" {}
variable "access_key" {}
variable "secret_key" {}
variable "region" {}
variable "site" {}
variable "bastion_init_sh" {}

variable "subnet" {
  description = "서브넷 CIDR 블록 리스트"
  type        = list(string)
}

variable "subnet_type" {
  description = "서브넷 타입 리스트 (PUBLIC/PRIVATE)"
  type        = list(string)
}

variable "usage_type" {
  description = "사용 타입 리스트"
  type        = list(string)
}

variable "support_vpc" {
  description = "VPC 지원 여부"
  type        = bool
  default     = true
}

 

- user_data.sh

# init-script.sh
#!/bin/bash
set -e

# basic packege install
sudo apt-get update -y
sudo apt-get install curl wget unzip git socat -y

cd /root/

# kubectl install
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
chmod +x kubectl
mv kubectl /usr/local/bin/

# ncp-iam-authenticator install
curl -o ncp-iam-authenticator -L https://github.com/NaverCloudPlatform/ncp-iam-authenticator/releases/latest/download/ncp-iam-authenticator_linux_amd64
chmod +x ncp-iam-authenticator
mv ncp-iam-authenticator /usr/local/bin/ncp-iam-authenticator

# kubeconfig create path
mkdir -p /root/.kube
touch /root/.kube/config

# env settings
export CLUSTER_UUID="${cluster_uuid}"
export CLUSTER_NAME="${cluster_name}"
export NCP_ACCESS_KEY="${NCP_ACCESS_KEY}"
export NCP_SECRET_KEY="${NCP_SECRET_KEY}"

# .ncloud/configure update
mkdir -p /root/.ncloud
cat << EOF > /root/.ncloud/configure
NCP_ACCESS_KEY=${NCP_ACCESS_KEY}
NCP_SECRET_KEY=${NCP_SECRET_KEY}
NCLOUD_API_GW=https://ncloud.apigw.ntruss.com
EOF

cat << EOF >> /etc/profile
export NCLOUD_ACCESS_KEY=${NCP_ACCESS_KEY}
export NCLOUD_SECRET_KEY=${NCP_SECRET_KEY}
export NCLOUD_API_GW=https://ncloud.apigw.ntruss.com
EOF

. /etc/profile

# kubeconfig update
ncp-iam-authenticator update-kubeconfig --clusterUuid ${cluster_uuid} --region KR

# Install krew
curl -L https://github.com/kubernetes-sigs/krew/releases/download/v0.4.4/krew-linux_amd64.tar.gz -o /root/krew-linux_amd64.tar.gz
tar zxvf krew-linux_amd64.tar.gz
./krew-linux_amd64 install krew
export PATH="$PATH:/root/.krew/bin"
echo 'export PATH="$PATH:/root/.krew/bin"' >> /etc/profile

# Install kube-ps1
echo 'source <(kubectl completion bash)' >> /root/.bashrc
echo 'alias k=kubectl' >> /root/.bashrc
echo 'complete -F __start_kubectl k' >> /root/.bashrc

git clone https://github.com/jonmosco/kube-ps1.git /root/kube-ps1
cat <<"EOT" >> /root/.bashrc
source /root/kube-ps1/kube-ps1.sh
KUBE_PS1_SYMBOL_ENABLE=false
function get_cluster_short() {
  echo "$1" | cut -d . -f1
}
KUBE_PS1_CLUSTER_FUNCTION=get_cluster_short
KUBE_PS1_SUFFIX=') '
PS1='$(kube_ps1)'$PS1
EOT

# Install krew plugin
kubectl krew install ctx ns get-all neat stern # ktop df-pv mtail tree
kubectl config rename-context "$(cat /root/.kube/config | grep current-context | awk '{print $2}')" "my-cluster"

 

실행 결과

 

- terraform 완료 메시지

 

- terraform output

패스워드와 key file 값은 별도 명령어로 확인

## root password 확인
terraform output -json root_password | jq -r '.root_password'

## public ip 확인
terraform output -raw public_ip

## 명령어 조합을 통한 bastion 서버 접속
apt install sshpass
sshpass -p $(terraform output -json root_password | jq -r '.root_password') ssh $(terraform output -raw public_ip)

 

- Bastion 서버에서 NKS 자원 조회