Categories
gcp Kubernetes

Install Halyard & Spinnaker on GKE

Create IAM Service Account

Spinnaker will use this IAM Service Account to access Google Cloud Storage (GCS) (storage.admin).

export SERVICE_ACCOUNT_NAME=spinnaker-itsmetommy-sa
export SERVICE_ACCOUNT_FILE=spinnaker-itsmetommy-sa.json
export SERVICE_ACCOUNT_DISPLAY_NAME="Spinnaker Account"
export PROJECT=$(gcloud info --format='value(config.project)')

gcloud --project ${PROJECT} iam service-accounts create \
  ${SERVICE_ACCOUNT_NAME} \
  --display-name ${SERVICE_ACCOUNT_DISPLAY_NAME}

sleep 10

SA_EMAIL=$(gcloud iam service-accounts list \
  --project=${PROJECT} \
  --filter="email ~ ${SERVICE_ACCOUNT_NAME}" \
  --format='value(email)')

gcloud --project ${PROJECT} projects add-iam-policy-binding ${PROJECT} \
  --role roles/storage.admin --member serviceAccount:${SA_EMAIL}

gcloud --project ${PROJECT} iam service-accounts keys create ${SERVICE_ACCOUNT_FILE} \
  --iam-account ${SA_EMAIL}

Create namespace

kubectl create ns spinnaker

Create k8s Service Account

This Service Account gives the Spinnaker deployment admin permissions to the cluster.

cat <<EOF | kubectl create -f -
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spinnaker-itsmetommy-sa
  namespace: spinnaker
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: spinnaker-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: spinnaker-itsmetommy-sa
  namespace: spinnaker
EOF

Create k8s kubeconfig

Create a kubeconfig file based on the above k8s Service Account (admin access to the Spinnaker namespace). We will use it when we configure our Halyard kubernetes provider below.

SERVICE_ACCOUNT_NAME=spinnaker-itsmetommy-sa
CONTEXT=$(kubectl config current-context)
SA_NAMESPACE=spinnaker
GCP_PROJECT=$(gcloud info --format='value(config.project)')
KUBECONFIG_FILE=spinnaker-itsmetommy-kubeconfig
NEW_CONTEXT=${SA_NAMESPACE}-sa

SECRET_NAME=$(kubectl get serviceaccount ${SERVICE_ACCOUNT_NAME} \
   --context ${CONTEXT} \
   --namespace ${SA_NAMESPACE} \
   -o jsonpath='{.secrets[0].name}')

TOKEN_DATA=$(kubectl get secret ${SECRET_NAME} \
   --context ${CONTEXT} \
   --namespace ${SA_NAMESPACE} \
   -o jsonpath='{.data.token}')

case "$(uname -s)" in
   Darwin*) TOKEN=$(echo ${TOKEN_DATA} | base64 -D);;
   Linux*) TOKEN=$(echo ${TOKEN_DATA} | base64 -d);;
   *) TOKEN=$(echo ${TOKEN_DATA} | base64 -d);;
   esac

kubectl config view --raw > ${KUBECONFIG_FILE}.full.tmp

# Switch working context to correct context
kubectl --kubeconfig ${KUBECONFIG_FILE}.full.tmp config use-context ${CONTEXT}

# Minify
kubectl --kubeconfig ${KUBECONFIG_FILE}.full.tmp \
   config view --flatten --minify > ${KUBECONFIG_FILE}.tmp

# Rename context
kubectl config --kubeconfig ${KUBECONFIG_FILE}.tmp \
   rename-context ${CONTEXT} ${NEW_CONTEXT}

# Create token user
kubectl config --kubeconfig ${KUBECONFIG_FILE}.tmp \
   set-credentials ${CONTEXT}-${SA_NAMESPACE}-token-user \
   --token ${TOKEN}

# Set context to use token user
kubectl config --kubeconfig ${KUBECONFIG_FILE}.tmp \
   set-context ${NEW_CONTEXT} --user ${CONTEXT}-${SA_NAMESPACE}-token-user

# Set context to correct namespace
kubectl config --kubeconfig ${KUBECONFIG_FILE}.tmp \
   set-context ${NEW_CONTEXT} --namespace ${SA_NAMESPACE}

# Flatten/minify kubeconfig
kubectl config --kubeconfig ${KUBECONFIG_FILE}.tmp \
   view --flatten --minify > ${KUBECONFIG_FILE}

# Remove tmp
rm ${KUBECONFIG_FILE}.full.tmp
rm ${KUBECONFIG_FILE}.tmp

Test the kubeconfig file.

kubectl --kubeconfig=spinnaker-itsmetommy-kubeconfig get pods

Create Halyard GCE Persistent Disk

gcloud compute disks create --size=200GB --zone=us-west1-a halyard-itsmetommy

Create k8s Halyard Deployment

Notice the Service Account spinnaker-itsmetommy-sa. It gives the Halyard deployment admin permissions to the cluster.

Get the latest stable version from https://gcr.io/spinnaker-marketplace/halyard.

kubectl apply -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: halyard
  namespace: spinnaker
  labels:
    app: halyard
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: halyard
  template:
    metadata:
      labels:
        app: halyard
    spec:
      nodeSelector:
        failure-domain.beta.kubernetes.io/zone: us-west1-a
      serviceAccountName: spinnaker-itsmetommy-sa
      containers:
      - name: halyard
        image: gcr.io/spinnaker-marketplace/halyard:1.42.0
        volumeMounts:
        - name: halyard-itsmetommy
          mountPath: "/home/spinnaker"
      securityContext:
        fsGroup: 1000
        runAsUser: 1000
      volumes:
      - name: halyard-itsmetommy
        gcePersistentDisk:
          pdName: halyard-itsmetommy
          fsType: ext4
EOF

Create GCS bucket for Spinnaker

We want persistent data generated by Spinnaker (e.g. pipeline definitions), so we will create a GCS bucket. We will configure the storage bucket in the next step.

BUCKET=[BUCKET_NAME]
gsutil mb gs://$BUCKET

Connect to the Halyard Pod

kubectl exec -it `kubectl get pods -n spinnaker -l app=halyard -o jsonpath="{.items[0].metadata.name}"` -n spinnaker -- sh

Configure Persistent Storage for Spinnaker

Because we want persistent data generated by Spinnaker (e.g. pipeline definitions), we will now configure Spinnaker to use the bucket we created in the previous step.

I decided to create a folder structure [PROVIDER]/[PROJECT] to keep things organized within the .secret directory..

# Make directory
GCP_PROJECT=[PROJECT_NAME]
mkdir -p /home/spinnaker/.secret/gcp/$GCP_PROJECT

Copy spinnaker-itsmetommy-sa.json to the Halyard pod.

GCP_PROJECT=[PROJECT_NAME]
HALYARD_POD=$(kubectl get pods -n spinnaker -l app=halyard -o jsonpath="{.items[0].metadata.name}")
kubectl cp spinnaker-itsmetommy-sa.json $HALYARD_POD:/home/spinnaker/.secret/gcp/$GCP_PROJECT -n spinnaker

# Verify
kubectl exec $HALYARD_POD -n spinnaker -- ls /home/spinnaker/.secret/gcp/$GCP_PROJECT

Configure the storage bucket.

GCP_PROJECT=[PROJECT_NAME]
BUCKET_LOCATION=us
BUCKET=spinnaker-itsmetommy
SERVICE_ACCOUNT_FILE=/home/spinnaker/.secret/gcp/$GCP_PROJECT/spinnaker-itsmetommy-sa.json

hal config storage gcs edit \
  --project $GCP_PROJECT \
  --bucket-location $BUCKET_LOCATION \
  --bucket $BUCKET \
  --json-path $SERVICE_ACCOUNT_FILE \
\
&& hal config storage edit --type gcs

Configure Kubernetes Provider

Copy spinnaker-itsmetommy-kubeconfig to the Halyard pod.

GCP_PROJECT=[PROJECT_NAME]
HALYARD_POD=$(kubectl get pods -n spinnaker -l app=halyard -o jsonpath="{.items[0].metadata.name}")
kubectl cp spinnaker-itsmetommy-kubeconfig $HALYARD_POD:/home/spinnaker/.secret/gcp/$GCP_PROJECT -n spinnaker

Configure kubeconfig and Cloud Provider.

GCP_PROJECT=[PROJECT_NAME]
SA_NAMESPACE=spinnaker
ACCOUNT_NAME=k8s-itsmetommy
KUBECONFIG_FILE=/home/spinnaker/.secret/gcp/$GCP_PROJECT/spinnaker-itsmetommy-kubeconfig

# Enable the Kubernetes cloud provider
hal config provider kubernetes enable

# Add the account
hal config provider kubernetes account add $ACCOUNT_NAME \
  --provider-version v2 \
  --kubeconfig-file $KUBECONFIG_FILE \
  --only-spinnaker-managed true \
  --namespaces $SA_NAMESPACE

# Verify
hal config provider kubernetes account list

Configure Artifacts

Within Spinnaker, ‘artifacts’ are consumable references to items that live outside of Spinnaker (for example, a file in a git repository or a file in an S3 bucket are two examples of artifacts). This feature must be explicitly turned on.

hal config features edit --artifacts true

Configure Distributed

Halyard deploys each of Spinnaker’s microservices separately. This is highly recommended for use in production.

Distributed installations are for development orgs with large resource footprints, and for those who can’t afford downtime during Spinnaker updates.

# Distributed installations are for development orgs with large resource footprints,
# and for those who can’t afford downtime during Spinnaker updates.
ACCOUNT_NAME=k8s-itsmetommy
hal config deploy edit --type distributed --account-name $ACCOUNT_NAME

Deploy Spinnaker

Choose Spinnaker version

# latest
hal config version edit --version $(hal version latest -q)

# Or you can specify the version
hal version list
export VERSION=1.23.0
hal config version edit --version $VERSION

Specify namespace (optional)

The default namespace is spinnaker.

hal config deploy edit --location [NAMESPACE]

Install Spinnaker

hal deploy apply

Watch

The complete spinnaker installation will take ~10 minutes.

watch kubectl get pods -n spinnaker

Connect to Spinnaker

  • Deck: Management UI for Spinnaker
  • Gate: Spinnaker API Gateway
NAMESPACE=spinnaker
DECK_POD=$(kubectl -n ${NAMESPACE} get pod -l cluster=spin-deck -ojsonpath='{.items[0].metadata.name}')
GATE_POD=$(kubectl -n ${NAMESPACE} get pod -l cluster=spin-gate -ojsonpath='{.items[0].metadata.name}')
kubectl -n ${NAMESPACE} port-forward ${DECK_POD} 9000 &
kubectl -n ${NAMESPACE} port-forward ${GATE_POD} 8084 &

View jobs.

jobs
[1]  - running    kubectl -n ${NAMESPACE} port-forward ${DECK_POD} 9000
[2]  + running    kubectl -n ${NAMESPACE} port-forward ${GATE_POD} 8084

Open browser.

open http://localhost:9000

Clean up

# Kill jobs
kill %1 %2

# Delete Spinnaker deployment
kubectl exec -it `kubectl get pods -n spinnaker -l app=halyard -o jsonpath="{.items[0].metadata.name}"` -n spinnaker -- sh
hal deploy clean

# Delete IAM Service Account
export SERVICE_ACCOUNT_NAME=spinnaker-itsmetommy-sa
export PROJECT=$(gcloud info --format='value(config.project)')
gcloud --project ${PROJECT} iam service-accounts delette ${SERVICE_ACCOUNT_NAME}
rm spinnaker-itsmetommy-sa.json

# Delete k8s kubeconfig file
rm spinnaker-itsmetommy-kubeconfig

# Delete Halyard deployment
kubectl delete deployment halyard -n spinnaker

# Delete GCE Persistent Disk
gcloud compute disks delete --zone=us-west1-a halyard-itsmetommy

# Delete bucket
gsutil rm -r gs://[BUCKET_NAME]

By Tommy Elmesewdy

DevOps Engineer