Chapter 3.6 - Continuous deployment of the model with the CI/CD pipeline¶

# Set the Kubernetes Cluster permissions for the Google Service Account
gcloud projects add-iam-policy-binding $GCP_PROJECT_ID \
    --member="serviceAccount:google-service-account@${GCP_PROJECT_ID}.iam.gserviceaccount.com" \
    --role="roles/container.developer"

name: MLOps

on:
  # Runs on pushes targeting main branch
  push:
    branches:
      - main

  # Runs on pull requests
  pull_request:

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  train-report-publish-and-deploy:
    permissions: write-all
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: pip
      - name: Install dependencies
        run: pip install --requirement requirements-freeze.txt
      - name: Login to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}'
      - name: Train model
        run: dvc repro --pull
      - name: Setup CML
        if: github.event_name == 'pull_request'
        uses: iterative/setup-cml@v2
        with:
          version: '0.20.0'
      - name: Create CML report
        if: github.event_name == 'pull_request'
        env:
          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Fetch all other Git branches
          git fetch --depth=1 origin main:main

          # Add title to the report
          echo "# Experiment Report (${{ github.sha }})" >> report.md

          # Compare parameters to main branch
          echo "## Params workflow vs. main" >> report.md
          dvc params diff main --md >> report.md

          # Compare metrics to main branch
          echo "## Metrics workflow vs. main" >> report.md
          dvc metrics diff main --md >> report.md

          # Compare plots (images) to main branch
          dvc plots diff main

          # Create plots
          echo "## Plots" >> report.md

          # Create training history plot
          echo "### Training History" >> report.md
          echo "#### main" >> report.md
          echo '![](./dvc_plots/static/main_evaluation_plots_training_history.png "Training History")' >> report.md
          echo "#### workspace" >> report.md
          echo '![](./dvc_plots/static/workspace_evaluation_plots_training_history.png "Training History")' >> report.md

          # Create predictions preview
          echo "### Predictions Preview" >> report.md
          echo "#### main" >> report.md
          echo '![](./dvc_plots/static/main_evaluation_plots_pred_preview.png "Predictions Preview")' >> report.md
          echo "#### workspace" >> report.md
          echo '![](./dvc_plots/static/workspace_evaluation_plots_pred_preview.png "Predictions Preview")' >> report.md

          # Create confusion matrix
          echo "### Confusion Matrix" >> report.md
          echo "#### main" >> report.md
          echo '![](./dvc_plots/static/main_evaluation_plots_confusion_matrix.png "Confusion Matrix")' >> report.md
          echo "#### workspace" >> report.md
          echo '![](./dvc_plots/static/workspace_evaluation_plots_confusion_matrix.png "Confusion Matrix")' >> report.md

          # Publish the CML report
          cml comment update --target=pr --publish report.md
      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}
          username: _json_key
          password: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
      - name: Import the BentoML model
        if: github.ref == 'refs/heads/main'
        run: bentoml models import model/celestial_bodies_classifier_model.bentomodel
      - name: Build the BentoML model artifact
        if: github.ref == 'refs/heads/main'
        run: bentoml build src
      - name: Containerize and publish the BentoML model artifact Docker image
        if: github.ref == 'refs/heads/main'
        run: |
          # Containerize the Bento
          bentoml containerize celestial_bodies_classifier:latest \
            --image-tag ${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}/celestial-bodies-classifier:latest \
            --image-tag ${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}/celestial-bodies-classifier:${{ github.sha }}
          # Push the container to the Container Registry
          docker push --all-tags ${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}/celestial-bodies-classifier
      - name: Get Google Cloud's Kubernetes credentials
        if: github.ref == 'refs/heads/main'
        uses: google-github-actions/get-gke-credentials@v2
        with:
          cluster_name: ${{ secrets.GCP_K8S_CLUSTER_NAME }}
          location: ${{ secrets.GCP_K8S_CLUSTER_ZONE }}
      - name: Update the Kubernetes deployment
        if: github.ref == 'refs/heads/main'
        run: |
          yq -i '.spec.template.spec.containers[0].image = "${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}/celestial-bodies-classifier:${{ github.sha }}"' kubernetes/deployment.yaml
      - name: Deploy the model on Kubernetes
        if: github.ref == 'refs/heads/main'
        run: |
          kubectl apply \
            -f kubernetes/deployment.yaml \
            -f kubernetes/service.yaml

# Show the differences with Git
git diff .github/workflows/mlops.yaml

diff --git a/.github/workflows/mlops.yaml b/.github/workflows/mlops.yaml
index 1fa989b..6d479ef 100644
--- a/.github/workflows/mlops.yaml
+++ b/.github/workflows/mlops.yaml
@@ -13,7 +13,7 @@ on:
   workflow_dispatch:

 jobs:
-  train-report-and-publish:
+  train-report-publish-and-deploy:
     permissions: write-all
     runs-on: ubuntu-latest
     steps:
@@ -106,3 +106,43 @@ jobs:

           # Push the container to the Container Registry
           docker push --all-tags ${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}/celestial-bodies-classifier
+      - name: Get Google Cloud's Kubernetes credentials
+        if: github.ref == 'refs/heads/main'
+        uses: google-github-actions/get-gke-credentials@v2
+        with:
+          cluster_name: ${{ secrets.GCP_K8S_CLUSTER_NAME }}
+          location: ${{ secrets.GCP_K8S_CLUSTER_ZONE }}
+      - name: Update the Kubernetes deployment
+        if: github.ref == 'refs/heads/main'
+        run: |
+          yq -i '.spec.template.spec.containers[0].image = "${{ secrets.GCP_CONTAINER_REGISTRY_HOST }}/celestial-bodies-classifier:${{ github.sha }}"' kubernetes/deployment.yaml
+      - name: Deploy the model on Kubernetes
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl apply \
+            -f kubernetes/deployment.yaml \
+            -f kubernetes/service.yaml

# Add the file
git add .gitlab/agents/k8s-agent/config.yaml

# Commit the changes
git commit -m "Enable Kubernetes integration with GitLab"

# Push the changes
git push

helm repo add gitlab https://charts.gitlab.io
helm repo update
helm upgrade --install XXX gitlab/gitlab-agent \
    --namespace XXX \
    --create-namespace \
    --set image.tag=XXX \
    --set config.token=XXX \
    --set config.kasAddress=XXX

"gitlab" has been added to your repositories
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "gitlab" chart repository
Update Complete. ⎈Happy Helming!⎈
Release "k8s-agent" does not exist. Installing it now.
NAME: k8s-agent
LAST DEPLOYED: Tue Aug 15 13:59:01 2023
NAMESPACE: gitlab-agent-k8s-agent
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Thank you for installing gitlab-agent.

Your release is named k8s-agent.

stages:
  - train
  - report
  - publish
  - deploy

variables:
  # Change pip's cache directory to be inside the project directory since we can
  # only cache local items.
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  # https://dvc.org/doc/user-guide/troubleshooting?tab=GitLab-CI-CD#git-shallow
  GIT_DEPTH: "0"
  # Set the path to Google Service Account key for DVC - https://dvc.org/doc/command-reference/remote/add#google-cloud-storage
  GOOGLE_APPLICATION_CREDENTIALS: "${CI_PROJECT_DIR}/google-service-account-key.json"
  # Environment variable for CML
  REPO_TOKEN: $GITLAB_PAT

train:
  stage: train
  image: python:3.11
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  before_script:
    # Set the Google Service Account key
    - echo "${GOOGLE_SERVICE_ACCOUNT_KEY}" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
    # Create the virtual environment for caching
    - python3.11 -m venv .venv
    - source .venv/bin/activate
  script:
    # Install dependencies
    - pip install --requirement requirements-freeze.txt
    # Run the experiment
    - dvc repro --pull
  cache:
    paths:
      # Pip's cache doesn't store the Python packages
      # https://pip.pypa.io/en/stable/reference/pip_install/#caching
      - .cache/pip
      - .venv/

report:
  stage: report
  image: iterativeai/cml:0-dvc3-base1
  needs:
    - train
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  before_script:
    # Set the Google Service Account key
    - echo "${GOOGLE_SERVICE_ACCOUNT_KEY}" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
  script:
    - |
      # Fetch the experiment changes
      dvc pull

      # Fetch all other Git branches
      git fetch --depth=1 origin main:main

      # Add title to the report
      echo "# Experiment Report (${CI_COMMIT_SHA})" >> report.md

      # Compare parameters to main branch
      echo "## Params workflow vs. main" >> report.md
      dvc params diff main --md >> report.md

      # Compare metrics to main branch
      echo "## Metrics workflow vs. main" >> report.md
      dvc metrics diff main --md >> report.md

      # Compare plots (images) to main branch
      dvc plots diff main

      # Create plots
      echo "## Plots" >> report.md

      # Create training history plot
      echo "### Training History" >> report.md
      echo "#### main" >> report.md
      echo '![](./dvc_plots/static/main_evaluation_plots_training_history.png "Training History")' >> report.md
      echo "#### workspace" >> report.md
      echo '![](./dvc_plots/static/workspace_evaluation_plots_training_history.png "Training History")' >> report.md

      # Create predictions preview
      echo "### Predictions Preview" >> report.md
      echo "#### main" >> report.md
      echo '![](./dvc_plots/static/main_evaluation_plots_pred_preview.png "Predictions Preview")' >> report.md
      echo "#### workspace" >> report.md
      echo '![](./dvc_plots/static/workspace_evaluation_plots_pred_preview.png "Predictions Preview")' >> report.md

      # Create confusion matrix
      echo "### Confusion Matrix" >> report.md
      echo "#### main" >> report.md
      echo '![](./dvc_plots/static/main_evaluation_plots_confusion_matrix.png "Confusion Matrix")' >> report.md
      echo "#### workspace" >> report.md
      echo '![](./dvc_plots/static/workspace_evaluation_plots_confusion_matrix.png "Confusion Matrix")' >> report.md

      # Publish the CML report
      cml comment update --target=pr --publish report.md

publish:
  stage: publish
  image: python:3.11
  needs:
    - train
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  services:
    - docker:25.0.3-dind
  variables:
    DOCKER_HOST: "tcp://docker:2375"
  before_script:
    # Install Docker
    - apt-get update
    - apt-get install --yes ca-certificates curl
    - install -m 0755 -d /etc/apt/keyrings
    - curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
    - chmod a+r /etc/apt/keyrings/docker.asc
    - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
    - apt-get update
    - apt-get install --yes docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
    # Set the Google Service Account key
    - echo "${GOOGLE_SERVICE_ACCOUNT_KEY}" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
    # Login to the Google Container Registry
    - cat "${GOOGLE_APPLICATION_CREDENTIALS}" | docker login -u _json_key --password-stdin "${GCP_CONTAINER_REGISTRY_HOST}"
    # Create the virtual environment for caching
    - python3.11 -m venv .venv
    - source .venv/bin/activate
  script:
    # Install dependencies
    - pip install --requirement requirements-freeze.txt
    # Pull the BentoML model artifact
    - dvc pull model/celestial_bodies_classifier_model.bentomodel
    # Import the BentoML model
    - bentoml models import model/celestial_bodies_classifier_model.bentomodel
    # Build the BentoML model artifact
    - bentoml build src
    # Containerize and publish the BentoML model artifact Docker image
    - |
      bentoml containerize celestial_bodies_classifier:latest \
        --image-tag $GCP_CONTAINER_REGISTRY_HOST/celestial-bodies-classifier:latest \
        --image-tag $GCP_CONTAINER_REGISTRY_HOST/celestial-bodies-classifier:${CI_COMMIT_SHA}
    # Push the container to the Container Registry
    - docker push --all-tags $GCP_CONTAINER_REGISTRY_HOST/celestial-bodies-classifier
  cache:
    paths:
      # Pip's cache doesn't store the Python packages
      # https://pip.pypa.io/en/stable/reference/pip_install/#caching
      - .cache/pip
      - .venv/

deploy:
  stage: deploy
  image: alpine
  needs:
    - publish
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  before_script:
    # Install kubectl and yq
    - apk add --no-cache kubectl yq
    # Switch to the right Kubernetes context
    - kubectl config use-context ${CI_PROJECT_PATH}:k8s-agent
    - kubectl get nodes
  script:
    # Update the Kubernetes deployment
    - yq -i '.spec.template.spec.containers[0].image = "${GCP_CONTAINER_REGISTRY_HOST}/celestial-bodies-classifier:${CI_COMMIT_SHA}"' kubernetes/deployment.yaml
    # Deploy the model on Kubernetes
    - kubectl apply -f kubernetes/deployment.yaml -f kubernetes/service.yaml

# Show the differences with Git
git diff .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dbf3b25..7dcdfe7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,8 @@
 stages:
   - train
   - report
   - publish
+  - deploy

 variables:
   # Change pip's cache directory to be inside the project directory since we can
@@ -95,3 +97,73 @@ report:

       - .cache/pip
       - .venv/
+
+deploy:
+  stage: deploy
+  image: alpine
+  needs:
+    - publish
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+  before_script:
+    # Install kubectl and yq
+    - apk add --no-cache kubectl yq
+    # Switch to the right Kubernetes context
+    - kubectl config use-context ${CI_PROJECT_PATH}:k8s-agent
+    - kubectl get nodes
+  script:
+    # Update the Kubernetes deployment
+    - yq -i '.spec.template.spec.containers[0].image = "${GCP_CONTAINER_REGISTRY_HOST}/celestial-bodies-classifier:${CI_COMMIT_SHA}"' kubernetes/deployment.yaml
+    # Deploy the model on Kubernetes
+    - kubectl apply -f kubernetes/deployment.yaml -f kubernetes/service.yaml

On branch main
Your branch is up to date with 'origin/main'.

Changes to be committed:
(use "git restore --staged <file>..." to unstage)
    modified:   .github/workflows/mlops.yaml

On branch main
Your branch is up to date with 'origin/main'.

Changes to be committed:
(use "git restore --staged <file>..." to unstage)
    modified:   .gitlab-ci.yml