Ansible NGINX Plus Ingress Controller 배포 자동화 – OpenShift

ansible-nic-deploy/
├── ansible.cfg                    # Ansible 설정 파일
├── site.yaml                      # 메인 플레이북 (배포)
├── cleanup.yaml                   # 삭제 플레이북 (안전장치 적용)
├── license.jwt                    # F5 NGINX 라이선스 파일 (Git 제외)
├── inventory/
│   └── hosts.yaml                 # 대상 클러스터 정의
├── tasks/
│   ├── set_common_facts.yaml      # kubectl/helm 인증 옵션 팩트 설정
│   ├── pre_check.yaml             # 도구 및 라이센스 파일 사전 검증
│   ├── setup_namespace_and_secrets.yaml  # 네임스페이스, 시크릿, SCC 구성
│   └── deploy_nic.yaml            # Helm을 통한 NIC 배포
├── vars/
│   ├── nic_vars.yaml              # 배포 관련 주요 변수
│   └── vault.yaml.example         # Ansible Vault 예시 파일
├── files/
│   ├── npic-values.yaml           # NIC Helm Chart 기본 Values
│   └── scc.yaml                   # OpenShift SCC 정의
└── kubeconfigs/
    ├── ansible-ocp.yaml           # Kubeconfig 파일
    └── README.md                  # Kubeconfig 관리 가이드

[defaults]
interpreter_python = auto_silent
retry_files_enabled = False
inventory = ./inventory/hosts.yaml

all:
  children:
    ocp_clusters:
      vars:
        ansible_connection: local # 모든 클러스터 작업은 로컬(kubectl/helm)에서 실행
      hosts:
        ansible-ocp:
          cluster_api_url: "https://api.ansible-ocp.devopssong.site:6443"
          kubeconfig_path: "{{ playbook_dir }}/kubeconfigs/ansible-ocp.yaml"
          # cluster_token을 사용할 경우 아래와 같이 호스트별 Vault 변수를 매핑합니다.
          # cluster_token: "{{ vault_token_ansible_ocp }}"

cluster_auth_method: kubeconfig
cluster_validate_certs: false
cluster_token: "{{ vault_cluster_token | default('') }}"
nic_namespace: nginx-ingress
helm_release_name: npic
helm_chart_repo_name: nginx-stable
helm_chart_repo_url: https://helm.nginx.com/stable
helm_chart_name: nginx-stable/nginx-ingress
helm_values_file: "{{ playbook_dir }}/files/npic-values.yaml"
# 업그레이드 시 이 값만 변경하세요.
# Chart 버전에 호환되는 NIC 버전이 내장되어 있습니다. (예: chart 2.5.2 → app 5.4.2)
# npic-values.yaml의 이미지 설정과 함께 관리하세요.
helm_chart_version: "2.5.2" # nginx-ingress chart version
nic_custom_values:
  controller.replicaCount: 2
  controller.service.type: "ClusterIP"
  controller.ingressClass.name: "nginx"
  # controller.resources.requests.cpu: "100m"
  # controller.resources.requests.memory: "128Mi"

---
- name: NIC Deployment on OpenShift
  hosts: ocp_clusters
  gather_facts: false
  vars_files:
    - vars/nic_vars.yaml
    # - vars/vault.yaml  # Token 인증 사용 시 주석 해제
  
  tasks:
    - name: Include common facts setup
      ansible.builtin.import_tasks: tasks/set_common_facts.yaml
      tags: always
      
    - name: Include pre-check tasks
      ansible.builtin.import_tasks: tasks/pre_check.yaml
      tags: precheck
      
    - name: Include namespace and secrets setup tasks
      ansible.builtin.import_tasks: tasks/setup_namespace_and_secrets.yaml
      tags: setup
      
    - name: Include NIC Helm deploy tasks
      ansible.builtin.import_tasks: tasks/deploy_nic.yaml
      tags: deploy

controller:
  # 1. NGINX Plus Ingress Controller & License/Registry Configuration
  nginxplus: true
  image:
    repository: private-registry.nginx.com/nginx-ic/nginx-plus-ingress
  serviceAccount:
    name: nginx-ingress
    imagePullSecretsNames:
      - regcred
      
  # 2. Performance & Protocol Configuration (NGINX ConfigMap Entries)
  config:
    entries:
      access-log-off: 'False'
      keepalive: '16'
      log-format: '$remote_addr - $remote_user [$time_local]  "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'
      real-ip-header: proxy_protocol
      proxy-protocol: 'True'
      set-real-ip-from: 192.168.0.0/16
      worker-connections: '4096'
      worker-rlimit-nofile: '65535'
  enableSnippets: true
  
  # 3. Graceful Shutdown Configuration
  terminationGracePeriodSeconds: 45
  lifecycle:
    preStop:
      exec:
        command:
          - /bin/sh
          - -c
          - sleep 40
          
  # 4. Monitoring & Telemetry (Prometheus & Status API)
  nginxStatus:
    allowCidrs: 0.0.0.0/0
  pod:
    annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/port: '9113'
      prometheus.io/scheme: http
  reportIngressStatus:
    leaderElectionLockName: nginx-ingress-leader
    
  # 5. Network Service & Readiness Port Mapping
  replicaCount: 2
  service:
    type: ClusterIP
    extraLabels:
      app: ingresslink
    customPorts:
      - name: readiness-port
        port: 8081
        targetPort: 8081
        protocol: TCP

---
- name: "[Common] Set kubectl auth options"
  ansible.builtin.set_fact:
    kubectl_auth_opts: >-
      {{
        '--kubeconfig ' + kubeconfig_path
        if cluster_auth_method == 'kubeconfig'
        else '--server ' + cluster_api_url + ' --token ' + cluster_token + ( ' --insecure-skip-tls-verify' if not cluster_validate_certs | bool else '' )
      }}
      
- name: "[Common] Set helm auth options"
  ansible.builtin.set_fact:
    helm_auth_opts: >-
      {{
        '--kubeconfig ' + kubeconfig_path
        if cluster_auth_method == 'kubeconfig'
        else '--kube-apiserver ' + cluster_api_url + ' --kube-token ' + cluster_token + ( ' --kube-insecure-skip-tls-verify' if not cluster_validate_certs | bool else '' )
      }}

---
- name: "[PreCheck] Check kubectl is installed"
  ansible.builtin.command: kubectl version --client
  register: kubectl_check
  changed_when: false
  
- name: "[PreCheck] Check helm is installed"
  ansible.builtin.command: helm version --short
  register: helm_check
  changed_when: false
  
- name: "[PreCheck] Check oc is installed"
  ansible.builtin.command: oc version --client
  register: oc_check
  changed_when: false
  
- name: "[PreCheck] Show tool versions"
  ansible.builtin.debug:
    msg:
      - "kubectl: {{ kubectl_check.stdout }}"
      - "helm: {{ helm_check.stdout }}"
      - "oc: {{ oc_check.stdout }}"
      
- name: "[PreCheck] Check license.jwt file existence"
  ansible.builtin.stat:
    path: "{{ playbook_dir }}/license.jwt"
  register: license_file
  
- name: "[PreCheck] Fail if license.jwt is missing"
  ansible.builtin.assert:
    that:
      - license_file.stat.exists
    fail_msg: "license.jwt not found at {{ playbook_dir }}/license.jwt."
    success_msg: "license.jwt found"
    
- name: "[PreCheck] Test cluster connection via kubeconfig"
  ansible.builtin.command: >
    kubectl cluster-info
    --kubeconfig {{ kubeconfig_path }}
  register: cluster_info_kubeconfig
  changed_when: false
  when: cluster_auth_method == 'kubeconfig'
  
- name: "[PreCheck] Test cluster connection via token"
  ansible.builtin.command: >
    kubectl cluster-info
    --server {{ cluster_api_url }}
    --token {{ cluster_token }}
    --insecure-skip-tls-verify={{ not cluster_validate_certs }}
  register: cluster_info_token
  changed_when: false
  when: cluster_auth_method == 'token'
  no_log: true

---
- name: "[Setup] Create NIC namespace"
  ansible.builtin.command: >
    kubectl create namespace {{ nic_namespace }}
    {{ kubectl_auth_opts }}
  register: ns_result
  changed_when: ns_result.rc == 0
  failed_when:
    - ns_result.rc != 0
    - "'already exists' not in ns_result.stderr"
    
- name: "[Setup] Create or update license-token secret (Declarative)"
  ansible.builtin.shell: >
    kubectl create secret generic license-token
    --from-file=license.jwt={{ playbook_dir }}/license.jwt
    --type=nginx.com/license
    -n {{ nic_namespace }}
    --dry-run=client -o yaml {{ kubectl_auth_opts }} | kubectl apply -f - {{ kubectl_auth_opts }}
  register: license_secret_result
  changed_when: "'configured' in license_secret_result.stdout or 'created' in license_secret_result.stdout"
  
- name: "[Setup] Read license.jwt content for regcred"
  ansible.builtin.slurp:
    src: "{{ playbook_dir }}/license.jwt"
  register: license_jwt_b64
  no_log: true
  when: create_nginx_registry_secret | default(true) | bool
  
- name: "[Setup] Create or update docker-registry secret (regcred - Declarative)"
  ansible.builtin.shell: >
    kubectl create secret docker-registry regcred
    --docker-server=private-registry.nginx.com
    --docker-username={{ license_jwt_b64['content'] | b64decode | trim }}
    --docker-password=none
    -n {{ nic_namespace }}
    --dry-run=client -o yaml {{ kubectl_auth_opts }} | kubectl apply -f - {{ kubectl_auth_opts }}
  register: regcred_secret_result
  changed_when: "'configured' in regcred_secret_result.stdout or 'created' in regcred_secret_result.stdout"
  when: create_nginx_registry_secret | default(true) | bool
  no_log: true
  
- name: "[Setup] Apply SCC (scc.yaml)"
  ansible.builtin.command: >
    oc apply -f {{ playbook_dir }}/files/scc.yaml
    {{ kubectl_auth_opts }}
  register: scc_apply_result
  changed_when: "'created' in scc_apply_result.stdout or 'configured' in scc_apply_result.stdout"
  
- name: "[Setup] Grant SCC to nginx-ingress-admin"
  ansible.builtin.command: >
    oc adm policy add-scc-to-user nginx-ingress-admin -z nginx-ingress -n {{ nic_namespace }}
    {{ kubectl_auth_opts }}
  register: scc_policy_result
  # oc adm policy는 이미 부여된 상태에서도 항상 "added" 메시지를 출력하므로 changed_when: false로 처리
  changed_when: false
  failed_when:
    - scc_policy_result.rc != 0

kind: SecurityContextConstraints
apiVersion: security.openshift.io/v1
metadata:
  name: nginx-ingress-admin
allowPrivilegedContainer: false
runAsUser:
  type: MustRunAs
  uid: 101
seLinuxContext:
  type: MustRunAs
fsGroup:
  type: MustRunAs
supplementalGroups:
  type: MustRunAs
allowHostNetwork: false
allowHostPID: false
allowHostPorts: false
allowHostDirVolumePlugin: false
allowHostIPC: false
readOnlyRootFilesystem: false
seccompProfiles:
- runtime/default
volumes:
  - secret
requiredDropCapabilities:
  - ALL
users:
  - 'system:serviceaccount:*:nginx-ingress'
allowedCapabilities:
  - NET_BIND_SERVICE

---
- name: "[Deploy] Add NGINX Helm chart repository"
  ansible.builtin.command: >
    helm repo add {{ helm_chart_repo_name }} {{ helm_chart_repo_url }}
  register: repo_add_result
  changed_when: "'already exists' not in repo_add_result.stdout"
  failed_when:
    - repo_add_result.rc != 0
    - "'already exists' not in repo_add_result.stdout"
    
- name: "[Deploy] Update Helm repositories"
  ansible.builtin.command: helm repo update
  changed_when: false
  
- name: "[Deploy] Reconcile NIC via Helm"
  ansible.builtin.command: >
    helm upgrade --install {{ helm_release_name }}
    {{ helm_chart_name }}
    -n {{ nic_namespace }}
    -f {{ helm_values_file }}
    {% if helm_chart_version | default('') != '' %}--version {{ helm_chart_version }}{% endif %}
    {% for key, value in (nic_custom_values | default({})).items() %}
    --set {{ key }}={{ value }}
    {% endfor %}
    --create-namespace
    --wait
    --timeout 5m
    {{ helm_auth_opts }}
  register: helm_apply_result
  changed_when: >
    helm_apply_result.rc == 0 and
    ('has been upgraded' in helm_apply_result.stdout or
     'has been installed' in helm_apply_result.stdout or
     'STATUS: deployed' in helm_apply_result.stdout)
    
- name: "[Deploy] Show Helm apply result"
  ansible.builtin.debug:
    msg: "{{ helm_apply_result.stdout_lines }}"
  when: helm_apply_result is defined
  
- name: "[Deploy] Wait for NIC pod to be Running"
  ansible.builtin.command: >
    kubectl rollout status deployment
    -l app.kubernetes.io/instance={{ helm_release_name }}
    -n {{ nic_namespace }}
    --timeout=120s
    {{ kubectl_auth_opts }}
  register: rollout_status
  changed_when: false

---
- name: NIC Cleanup on OpenShift
  hosts: ocp_clusters
  gather_facts: false
  vars_files:
    - vars/nic_vars.yaml
    
  tasks:
    - name: Include common facts setup
      ansible.builtin.import_tasks: tasks/set_common_facts.yaml
      tags: always
      
    - name: "[Cleanup] Safety check — confirm target"
      ansible.builtin.debug:
        msg:
          - "⚠️  About to DESTROY NIC on: {{ inventory_hostname }}"
          - "    Namespace: {{ nic_namespace }}"
          - "    Release:   {{ helm_release_name }}"
      tags: always
      
    - name: "[Cleanup] Abort if confirm_cleanup is not set"
      ansible.builtin.fail:
        msg: >
          Cleanup aborted. To proceed, run with:
          ansible-playbook cleanup.yaml -e confirm_cleanup=yes
      when: confirm_cleanup | default('no') != 'yes'
      tags: always
      
    - name: "[Cleanup] Uninstall NIC Helm release"
      ansible.builtin.command: >
        helm uninstall {{ helm_release_name }}
        -n {{ nic_namespace }}
        {{ helm_auth_opts }}
      register: helm_uninstall_result
      changed_when: helm_uninstall_result.rc == 0
      failed_when:
        - helm_uninstall_result.rc != 0
        - "'not found' not in helm_uninstall_result.stderr"
        
    - name: "[Cleanup] Delete generic secrets if namespace is preserved"
      ansible.builtin.command: >
        kubectl delete secret license-token regcred
        -n {{ nic_namespace }}
        --ignore-not-found=true
        {{ kubectl_auth_opts }}
      register: secrets_delete_result
      changed_when: "'deleted' in secrets_delete_result.stdout"
      when:
        - nic_namespace == 'kube-system' or nic_namespace == 'default'
        
    - name: "[Cleanup] Delete NIC namespace"
      ansible.builtin.command: >
        kubectl delete namespace {{ nic_namespace }}
        {{ kubectl_auth_opts }}
      register: ns_delete_result
      changed_when: ns_delete_result.rc == 0
      failed_when:
        - ns_delete_result.rc != 0
        - "'not found' not in ns_delete_result.stderr"
      when:
        - nic_namespace != 'kube-system'
        - nic_namespace != 'default'
        
    - name: "[Cleanup] Delete SCC"
      ansible.builtin.command: >
        oc delete scc nginx-ingress-admin
        {{ kubectl_auth_opts }}
      register: scc_delete_result
      changed_when: scc_delete_result.rc == 0
      failed_when:
        - scc_delete_result.rc != 0
        - "'not found' not in scc_delete_result.stderr"