GitOps와 ML 파이프라인이 만나는 지점

digits-training/
├── .gitlab-ci.yml
├── Dockerfile
├── pyproject.toml
├── uv.lock
├── src/
│   └── train.py
└── tests/
    └── test_train.py

ml-platform-gitops/
├── applications/
│   └── digits-ml.yaml
└── charts/
    └── digits-ml/
        ├── Chart.yaml
        ├── values.yaml
        ├── values-prod.yaml
        └── templates/
            ├── rayjob.yaml
            └── inferenceservice.yaml

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - when: never

.training_changes: &training_changes
  changes:
    - src/**/*
    - tests/**/*
    - pyproject.toml
    - uv.lock
    - Dockerfile
    - .gitlab-ci.yml

stages:
  - test
  - build
  - writeback

test-training-code:
  stage: test
  image: "${UV_TEST_IMAGE}"
  rules:
    - <<: *training_changes
  script:
    - uv sync --frozen --dev
    - uv run pytest -q
    - uv lock --check

build-training-image:
  stage: build
  image:
    name: "${BUILDKIT_IMAGE}"
    entrypoint: [""]
  variables:
    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
    HARBOR_REGISTRY: harbor.lab.example.com
    IMAGE_REPOSITORY: harbor.lab.example.com/mlops/digits-trainer
  needs:
    - test-training-code
  rules:
    - <<: *training_changes
  before_script:
    - mkdir -p "${HOME}/.docker"
    - |
      AUTH="$(printf '%s:%s' "${HARBOR_ROBOT_USER}" "${HARBOR_ROBOT_PASSWORD}" | base64 | tr -d '\n')"
      printf '{"auths":{"%s":{"auth":"%s"}}}\n' \
        "${HARBOR_REGISTRY}" "${AUTH}" > "${HOME}/.docker/config.json"
  script:
    - |
      buildctl-daemonless.sh build \
        --frontend dockerfile.v0 \
        --local context=. \
        --local dockerfile=. \
        --opt filename=Dockerfile \
        --output "type=image,name=${IMAGE_REPOSITORY}:${CI_COMMIT_SHA},push=true"
    - printf 'IMAGE_REPOSITORY=%s\nIMAGE_TAG=%s\n' "${IMAGE_REPOSITORY}" "${CI_COMMIT_SHA}" > image.env
  artifacts:
    reports:
      dotenv: image.env

training:
  image:
    repository: harbor.lab.example.com/mlops/digits-trainer
    tag: a1b2c3d4e5f6
  runId: a1b2c3d4e5f6
  source:
    commit: a1b2c3d4e5f6

serving:
  enabled: false
  model:
    storageUri: ""

metadata:
  name: {{ printf "%s-%s" .Release.Name .Values.training.runId | trunc 63 | trimSuffix "-" }}
  labels:
    app.kubernetes.io/name: {{ .Release.Name | quote }}
    mlops.example.com/source-commit: {{ .Values.training.source.commit | quote }}

write-training-state:
  stage: writeback
  image: alpine:3.21
  needs:
    - job: build-training-image
      artifacts: true
  rules:
    - <<: *training_changes
  resource_group: ml-platform-gitops-writeback
  variables:
    CHART_REPO_URL: https://gitlab.lab.example.com/mlops/ml-platform-gitops.git
    CHART_VALUES_FILE: charts/digits-ml/values.yaml
  before_script:
    - apk add --no-cache git yq
    - git config --global user.name "mlops-ci"
    - git config --global user.email "mlops-ci@lab.example.com"
  script:
    - |
      set -eu
      export GITLAB_USER=oauth2
      export GITLAB_PASSWORD="${CHART_REPO_TOKEN}"
      cat > /tmp/git-askpass <<'EOF'
      #!/bin/sh
      case "$1" in
        *Username*) printf '%s\n' "$GITLAB_USER" ;;
        *Password*) printf '%s\n' "$GITLAB_PASSWORD" ;;
      esac
      EOF
      chmod 700 /tmp/git-askpass
      export GIT_ASKPASS=/tmp/git-askpass
      export GIT_TERMINAL_PROMPT=0

      git clone "${CHART_REPO_URL}" gitops
      cd gitops
      export IMAGE_REPOSITORY IMAGE_TAG CI_COMMIT_SHA
      yq -i '.training.image.repository = strenv(IMAGE_REPOSITORY)' "${CHART_VALUES_FILE}"
      yq -i '.training.image.tag = strenv(IMAGE_TAG)' "${CHART_VALUES_FILE}"
      yq -i '.training.runId = strenv(CI_COMMIT_SHA)' "${CHART_VALUES_FILE}"
      yq -i '.training.source.commit = strenv(CI_COMMIT_SHA)' "${CHART_VALUES_FILE}"
      git add "${CHART_VALUES_FILE}"
      git diff --cached --quiet && exit 0
      git commit -m "train: deploy ${CI_COMMIT_SHA} [skip ci]"
      git push origin HEAD:main

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: digits-ml
  namespace: argocd
spec:
  project: ml-platform
  source:
    repoURL: https://gitlab.lab.example.com/mlops/ml-platform-gitops.git
    targetRevision: main
    path: charts/digits-ml
    helm:
      valueFiles:
        - values.yaml
  destination:
    server: https://kubernetes.default.svc
    namespace: mlops-demo
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true

# GitOps 상태
argocd app get digits-ml

# 배포된 RayJob과 이미지
kubectl get rayjob -n mlops-demo
kubectl get rayjob -n mlops-demo -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.rayClusterSpec.headGroupSpec.template.spec.containers[0].image}{"\n"}{end}'

# 학습 로그
kubectl get pods -n mlops-demo
kubectl logs -n mlops-demo -l ray.io/node-type=head --tail=200

import mlflow
import mlflow.sklearn
from mlflow.models import infer_signature

with mlflow.start_run() as run:
    # 학습과 평가 코드 생략
    mlflow.log_metric("accuracy", accuracy)
    mlflow.set_tags(
        {
            "validation_status": "pending",
            "source.git_commit": git_commit,
            "runtime.training_image": training_image,
        }
    )
    model_info = mlflow.sklearn.log_model(
        sk_model=model,
        artifact_path="model",
        signature=infer_signature(x_train, model.predict(x_train)),
        registered_model_name="digits-classifier",
    )
    print({"run_id": run.info.run_id, "model_uri": model_info.model_uri})

import os
from mlflow import MlflowClient

client = MlflowClient()
model_name = "digits-classifier"
candidate_version = os.environ["CANDIDATE_MODEL_VERSION"]
candidate = client.get_model_version(model_name, candidate_version)
run = client.get_run(candidate.run_id)

accuracy = run.data.metrics["accuracy"]
if accuracy < float(os.getenv("MIN_ACCURACY", "0.95")):
    client.set_model_version_tag(model_name, candidate_version, "validation_status", "failed")
    raise SystemExit(f"accuracy gate failed: {accuracy}")

client.set_model_version_tag(model_name, candidate_version, "validation_status", "passed")
client.set_registered_model_alias(model_name, "champion", candidate_version)

serving:
  enabled: true
  model:
    format: mlflow
    runtime: kserve-mlserver
    storageUri: s3://ml-models/digits/sha256-8dd4...b91/model
    mlflowModelVersion: "12"
    sourceRunId: 4c87...e19
  minReplicas: 1

{{- if .Values.serving.enabled }}
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: {{ .Release.Name }}-predictor
  annotations:
    mlops.example.com/mlflow-model-version: {{ .Values.serving.model.mlflowModelVersion | quote }}
    mlops.example.com/source-run-id: {{ .Values.serving.model.sourceRunId | quote }}
spec:
  predictor:
    minReplicas: {{ .Values.serving.minReplicas }}
    serviceAccountName: kserve-model-reader
    model:
      modelFormat:
        name: {{ .Values.serving.model.format }}
      runtime: {{ .Values.serving.model.runtime }}
      protocolVersion: v2
      storageUri: {{ .Values.serving.model.storageUri | quote }}
{{- end }}

helm template digits-ml ./charts/digits-ml -f ./charts/digits-ml/values.yaml
argocd app manifests digits-ml
kubectl get rayjob -n mlops-demo