setup production cluster

2021-08-30 20:21:19 +02:00 · 2021-08-30 20:21:19 +02:00 · 86036ed59d
parent cbb5431fa3
commit 86036ed59d
13 changed files with 194 additions and 85 deletions
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@ -65,25 +65,66 @@ pipelines:
            - docker push "gcr.io/${GCLOUD_PROJECT}/coturn-dns:${VERSION}"
          services:
            - docker
+
+
+      # - step: Deploy haproxy ingress controller
+      #     % helm install haproxy-ingress haproxy-ingress/haproxy-ingress\                 
+      # --create-namespace --namespace ingress-controller\
+      # --version 0.13.1\
+      # -f k8s/haproxy/haproxy-ingress-values.yaml
+
+
+      # - step: Deploy cert-manager
+      #     helm install \
+      # cert-manager jetstack/cert-manager \
+      # --namespace cert-manager \
+      # --create-namespace \
+      # --version v1.5.0 \
+      # --set installCRDs=true
+      #   $ kubectl apply -f k8s/cert-manager/cluster-issuer-production.yaml       
+
+
+
+      # - step: Deploy GCR credentials
+      # kubectl create secret docker-registry gcr-json-key \
+      #   --docker-server=gcr.io \
+      #   --docker-username=_json_key \
+      #   --docker-password="$(cat k8s/gcp.json)" \
+      #   --docker-email=any@valid.email
+      # kubectl patch serviceaccount default \
+      #  -p '{"imagePullSecrets": [{"name": "gcr-json-key"}]}'
+
+# Deploy nginx ingress controller
+# helm install nginx-ingress stable/nginx-ingress 
+
+# Deploy monitoring clusterissuer
+# kubectl apply -f k8s/monitoring/clusterissuer.yaml
+
+# Deploy monitoring certificate
+# kubectl apply -f k8s/monitoring/certificate.yaml
+
+# Deploy monitoring helm
+# helm install \
+# monitoring stable/prometheus-operator \
+# -f k8s/monitoring/helm-values.yaml \
+# --namespace monitoring \
+# --set grafana.adminPassword=jamkazamMonitoring
+
+
      - step:
          name: Deploy to K8s
-          deployment: production
+          deployment: staging
          script:
            - AUTOSCALER_IMAGE="gcr.io/$GCLOUD_PROJECT/autoscaler:prod-0.1.$BITBUCKET_BUILD_NUMBER"
            - COTURN_DNS_IMAGE="gcr.io/$GCLOUD_PROJECT/coturn-dns:prod-0.1.$BITBUCKET_BUILD_NUMBER"
            - sed -i "s|{{linode_autoscaler_image}}|$AUTOSCALER_IMAGE|g" k8s/linode-autoscaler/webrtc-be-autoscaler.yaml
            - sed -i "s|{{linode_autoscaler_image}}|$AUTOSCALER_IMAGE|g" k8s/linode-autoscaler/coturn-autoscaler.yaml
            - sed -i "s|{{coturn_dns_image}}|$COTURN_DNS_IMAGE|g" k8s/coturn-dns/coturn-dns.yaml
-            - pipe: atlassian/kubectl-run:1.1.2
-              variables:
-                KUBE_CONFIG: $KUBE_CONFIG
-                KUBECTL_COMMAND: 'apply'
-                RESOURCE_PATH: 'k8s/linode-autoscaler/'
-            - pipe: atlassian/kubectl-run:1.1.2
-              variables:
-                KUBE_CONFIG: $KUBE_CONFIG
-                KUBECTL_COMMAND: 'apply'
-                RESOURCE_PATH: 'k8s/external-dns/'
+            # - pipe: atlassian/kubectl-run:1.1.2
+            #   variables:
+            #     KUBE_CONFIG: $KUBE_CONFIG
+            #     KUBECTL_COMMAND: 'apply'
+            #     RESOURCE_PATH: 'k8s/linode-autoscaler/'
            - pipe: atlassian/kubectl-run:1.1.2
              variables:
                KUBE_CONFIG: $KUBE_CONFIG
--- a/k8s/cert-manager/certificate-production.yaml
+++ b/k8s/cert-manager/certificate-production.yaml
@ -0,0 +1,10 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: production
+spec:
+  secretName: production-certificate
+  issuerRef:
+    name: letsencrypt-production
+  dnsNames:
+  - video.jamkazam.com
--- a/k8s/coturn-dns/production-coturn-dns.yaml
+++ b/k8s/coturn-dns/production-coturn-dns.yaml
@ -0,0 +1,45 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coturn-dns
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: coturn-dns
+  template:
+    metadata:
+      labels:
+        app: coturn-dns
+    spec:
+      containers:
+        - name: coturn-dns
+          image: {{coturn_dns_image}}
+          env:
+          - name: AWS_ACCESS_KEY_ID
+            value: "AKIA2SXEHOQFBQRGCSST"
+          - name: AWS_SECRET_ACCESS_KEY
+            value: "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"
+          - name: PYTHONUNBUFFERED
+            value: "1"
+          - name: HOSTED_ZONE
+            value: "Z00156242SK162FEXDPVF"
+          - name: CLUSTER_ID
+            value: "29062"
+          - name: POOL_ID
+            value: "49934"
+          - name: LINODE_TOKEN
+            value: "a821bb97039cbd8b259e19ef9f7ea7a4e295a7399e00709fc27cad2b1f3742f4"
+          - name: COTURN_DOMAIN_NAME
+            value: "coturn.video.jamkazam.com"
+          resources:
+            requests:
+              memory: 32Mi
+            limits:
+              memory: 32Mi
--- a/k8s/coturn-dns/register-nodes.py
+++ b/k8s/coturn-dns/register-nodes.py
@ -4,11 +4,12 @@ import boto3
 import time
 import os

-HOSTED_ZONE=os.environ['HOSTED_ZONE'] #"Z00156242SK162FEXDPVF"
-CLUSTER_ID=os.environ['CLUSTER_ID'] #"29062"
-POOL_ID=os.environ['POOL_ID'] #"49934"
+HOSTED_ZONE=os.environ['HOSTED_ZONE'] 
+CLUSTER_ID=os.environ['CLUSTER_ID'] 
+POOL_ID=os.environ['POOL_ID'] 
 LINODE_TOKEN=os.environ['LINODE_TOKEN']
 TOKEN={"Authorization": "Bearer "+LINODE_TOKEN}
+COTURN_DOMAIN_NAME=os.environ['COTURN_DOMAIN_NAME']

 while(True):
    r = requests.get("https://api.linode.com/v4/lke/clusters/"+CLUSTER_ID+"/pools/"+POOL_ID, headers=TOKEN)
@ -17,7 +18,6 @@ while(True):

    for node in r.json()['nodes']:
        ip = requests.get("https://api.linode.com/v4/linode/instances/"+str(node['instance_id'])+"/ips", headers=TOKEN)
-        #print(ip.json())
        ips.append({'Value': ip.json()['ipv4']['public'][0]['address']})

    print("Node IPs: "+str(ips))
@ -30,7 +30,7 @@ while(True):
                                {
                                'Action': 'UPSERT',
                                'ResourceRecordSet': {
-                                    'Name': 'coturn.staging.video.jamkazam.com',
+                                    'Name': COTURN_DOMAIN_NAME,
                                    'Type': 'A',
                                    'TTL': 300,
                                    'ResourceRecords': ips
--- a/k8s/coturn-dns/staging-coturn-dns.yaml
+++ b/k8s/coturn-dns/staging-coturn-dns.yaml
--- a/k8s/external-dns/deployment.yaml
+++ b/k8s/external-dns/deployment.yaml
@ -1,68 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: external-dns
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: external-dns
-rules:
- apiGroups: [""]
-  resources: ["services","endpoints","pods"]
-  verbs: ["get","watch","list"]
- apiGroups: ["extensions","networking.k8s.io"]
-  resources: ["ingresses"]
-  verbs: ["get","watch","list"]
- apiGroups: [""]
-  resources: ["nodes"]
-  verbs: ["list","watch"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: external-dns-viewer
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: external-dns
-subjects:
- kind: ServiceAccount
-  name: external-dns
-  namespace: default
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: external-dns
-spec:
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: external-dns
-  template:
-    metadata:
-      labels:
-        app: external-dns
-    spec:
-      serviceAccountName: external-dns
-      containers:
-      - name: external-dns
-        image: k8s.gcr.io/external-dns/external-dns:v0.7.6
-        env:
-        - name: AWS_ACCESS_KEY_ID
-          value: "AKIA2SXEHOQFBQRGCSST"
-        - name: AWS_SECRET_ACCESS_KEY
-          value: "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"
-        args:
-        - --source=ingress
-        - --source=service
-        - --domain-filter=video.jamkazam.com # will make ExternalDNS see only the hosted zones matching provided domain, omit to process all available hosted zones
-        - --provider=aws
-        - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization
-        - --aws-zone-type=public # only look at public hosted zones (valid values are public, private or no value for both)
-        - --registry=txt
-        - --txt-owner-id=my-hostedzone-identifier
-      securityContext:
-        fsGroup: 65534
--- a/k8s/gcp.json
+++ b/k8s/gcp.json
@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "tough-craft-276813",
+  "private_key_id": "a8092b39b4eb391e8b1e8ace86d5c463e049e711",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCy18xh+H6vH4qJ\ns0x7syo8rK+dEgy/24dUTqPb54KfBmtXPdKuGCT/ZsoWAPqRhpmbYYe1Po9wNe6E\nXstVCvFq5ev2olJFzauy24UI6bWaXkQX/OHXLho/rn/EJPdcwBBQZ6mtrv+rgLWQ\nhiAHFMeaQSfwGrXeNnKWuT/PlJmDvliORjzm94r9fywzhArJq/lFNh0JWLTHfzVT\n6nhHIrOCQ+6IAszVerU6G7VfTAKoEaFS1OeLFwlUyhwc3SPm7ceLxBqz25APo3qA\nZFYyfLe43XbmKw1gta/QnpnPUtp3Wrm7sk9xy/maLx6xagVaUsGLNjWnZCjaPTkw\npe7FHU5XAgMBAAECggEADBP635ryo00UBByxy6Db92EKMydm6QYga5csBcvqzGaY\nlTm9orhKt1zvxPCn+3AFq7K4gYsKEN/zjckBHmswxrFkcDGiMMilEd01bNarxxMa\nsiwH7IpWh3p3cn20nvTxpRx7Hxm0dpaorGwCebfziv1ffx2urqUqs/cq0hANFhKF\n7bNYiTY6/9ZwWvcorpeu59UgJat2f12+aRUjj3Iu459UlRs6IhfXW2cWhMVHVylF\ng500i7sLrBLAlqGq8HnHkHUcB6sWnrWMBQ3wyqcEnORjVI0Oumaz1tphPEmxBy1n\n12arKrQ3N7Iij6mG/EX9Ha7J3tbFgb5Z9Xn3EObEAQKBgQDXBqm+HMEh35C7Jx7l\nhKdwRx87LhmBgDfGSxrNV0D/O8AFTPIuSDNeYi473AvUjsmnd6tQvtNFD6v8U16k\nRSwrwAr1eM4b8CIZ+nnMKt0ah96E8TyOBdp5Xfs18M4ZL9yddOpVrIVlDiQBIuHR\nZKvYvklxyxi5Ut6UtcNkKSl9VwKBgQDU7BBG//WeGC4N8e61pxfh+oBiNx6RoBt8\n++GPmksRwZYPnHqCtli5GX4UTQIrTAeAzbOzqe5t6G7yPqnJqKfPQnzZEXVu7d51\nFFIU7WAIUPs7AyNKDsWRDQ73q3M2EN3VqjyMX6DuUeTPfASjI8CCju0FtDtzqdm+\nSWDVLDcXAQKBgFRE1DkhY782sq3mAwHIHyateNvkkTJjYXhg7rwSufJNJE/ve+oP\nebI/oAbtkeVXoEf1ajpWzs19+tUEh06xnUH4HVNeaMgiL/smYp1VHxnKrbZEJIs0\nWA7AejcFjH/qdfdvXnb9Cbo09H9NgFpjrcVfrcDe622VwI1fPpf+Wbg5AoGBAIqo\nvKTwFU0CZCOStSi5CzWPw8GyMYcWZDBNfAPfsBl9HzNFbQbopvjL4C5qRApcNdqs\nmuVaubn7jxzUsA9ydO3lV5ao5vf5klBejmGwgESKMEGq9nVJD2I5xdCGZ74C1+RI\nO6wSrqPk0wRHuGFhbAHaAAMh70GQkAt6j8PjSnEBAoGBAK04V8fXPbCBxLoRfMbT\nBjeutWad36oTDuvLoIsMRM1vCF1oxpL+j4+7+hbupQ/UMcLvPN9RmwgJTjxOPN5b\nThwUn6UHfNWlb0pQrw764gMV+3EZgbEzx7pAi8QNEY5gLL0Qd/34eIm1exHuPJtM\n+MLbJDdyJ3PEZL9YOB1uKyC6\n-----END PRIVATE KEY-----\n",
+  "client_email": "ansible-sa@tough-craft-276813.iam.gserviceaccount.com",
+  "client_id": "104334872115406805719",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/ansible-sa%40tough-craft-276813.iam.gserviceaccount.com"
+}
--- a/k8s/monitoring/certificate.yaml
+++ b/k8s/monitoring/certificate.yaml
@ -8,7 +8,7 @@ spec:
  duration: 2160h # 90d
  renewBefore: 360h # 15d
  issuerRef:
-    name: letsencrypt-production
+    name: letsencrypt-monitoring
    kind: ClusterIssuer
  dnsNames:
  - monitoring.video.jamkazam.com
--- a/k8s/monitoring/clusterissuer.yaml
+++ b/k8s/monitoring/clusterissuer.yaml
@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-monitoring
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: victor.barba.martin@toptal.com
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource that will be used to store the account's private key.
+      name: issuer-account-key
+    # Add a single challenge solver, HTTP01 using nginx
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
--- a/k8s/monitoring/namespace.yaml
+++ b/k8s/monitoring/namespace.yaml
@ -0,0 +1 @@
+#TODO
--- a/k8s/prd-video-cluster-kubeconfig.yaml
+++ b/k8s/prd-video-cluster-kubeconfig.yaml
@ -0,0 +1,25 @@
+
+apiVersion: v1
+kind: Config
+preferences: {}
+
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EZ3hPREUxTURRd01Wb1hEVE14TURneE5qRTFNRFF3TVZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTDh0ClBtclBFM0ZGaUUrbm1OYUFvME00Zmg3UUpoSkdxYjNORTdYdHdtUkJ5S2RpWkQzUm5MYjdEa0RKQ2tiS3RaTnAKRWFKWFZmamk4MTl4N25vZzlLSlFGS1hNNFEwWWZsK0paNkY3WnhuOGZEeittYXhEUUVWWE1IZ3QzMVNPVUJydgpGRTVNb21hYnhtM25DdkRGM2IxMnlvNWFIVTYvdklhSHpSQWV2WkI1TFQ2RlY0blRxKzlVU1k4aVdIYjJDei92CmZ1aUd2TGE2N0tqVUdBL1dCeVREZitWSnV6cXUrSHdTZHFKU0lQYlNNTkJuaHFmNC9YV1Ayb3VJN1h5bEtnbG4KSHg2dGw2RVBYWXpVVWNaY215RUowYnhlNDdiZWFLcWoyc1l5cGNnTVNKSjM1UXNRLzU4MUdaRUpLTWFSUmpjMQpHTm1aWGozY0RPK2p0RFJkWjBFQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZKVHp2aC9GNkJHWkt0cDRJOTdRZWN4UGgxbUVNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDVmVBMEw4dFhTaWZEYWJqMlBnSEcvdkxQdXh6SXdJZXk3NVptVU12NlBLaGxNcE4zSApTcXUrUnhrVlM3S3pJejE1Z2RzWG4rZm9lZ2hvWUdtaVZWVmx0aTNZejVNSU5tTXZEZksxWTFjQ2xzWXNCdzBnCjE0SHpGRUoxaEExdXh3UGFKdzdVUDJyWE5LYWtKMHZVOVVpTnpxM0laWkFZT2p1TGVQLzQyeHhBVUNQdGhSZ28KTW9qTlhrdjVvVTlKS3FycDIvSmd1ZFJhVExoSEJGQUsxVDJrc3U2TVA5NU5qUTFyOWhBK0tieEVnbk93d2tSLwppV2hvUEpNNTdSWFF1ZzJOVzZaU29TRUFhdWN4Qy90UEo5bldwWk9TYkRReFNJYXJjdWdyMzlsWWZ1L1l5ak5kCnExUWtiODYzekxhZitxVmJEY0ladGRZQ3djTnN5RjRLcUF6awotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+    server: https://2c96c0af-61c1-4a4a-a7f7-3f5f4aa008fa.cpc2-us-central.linodelke.net:443
+  name: lke35025
+
+users:
+- name: lke35025-admin
+  user:
+    as-user-extra: {}
+    token: eyJhbGciOiJSUzI1NiIsImtpZCI6IkJkektZakZheTZHYVhMcjJ6YWlpdGpUQ1dxMWpEdlhCYi1JN05KNnBsVDAifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJsa2UtYWRtaW4tdG9rZW4tNTVuOWwiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoibGtlLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYWMxNjhkODQtZmNkOC00MTFiLTk5M2MtYzQyMjc4NTEwODU1Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmxrZS1hZG1pbiJ9.C-Co-E_coNyRqtRImZoR7IpPgNyZUYcF5ERMkjRWNS1r5mb7aETeduZkUQ6nEEgbke50tjt3MgHV0VFZr8rx22JiXMI6K6KmhPRR4aCznf9jFOi0Uja7sRLAeDDuVqn7Z0a858gTswkGPlBZCFyBzKUj6NGGpOpTzYb1Y8-AZF6ns8DaVntp6n1yFQXtQ3hXfAzfaF_JRREjn1mTJsV9025rlyGrGJ5tNGgnlQsIOYTs3GxFvE-LImw2041MLc875F21N78YzzaMC7ujh7PsXw2UVfHZknHRUwojd9see7C0kGf0W7xPSPtxXyEC1Cyp7YPGHSbUPulI0oiKNATCMA
+
+contexts:
+- context:
+    cluster: lke35025
+    namespace: default
+    user: lke35025-admin
+  name: lke35025-ctx
+
+current-context: lke35025-ctx
--- a/k8s/stg-video-cluster-kubeconfig.yaml
+++ b/k8s/stg-video-cluster-kubeconfig.yaml
--- a/terraform/lke.tf
+++ b/terraform/lke.tf
@ -22,3 +22,27 @@ resource "linode_lke_cluster" "my-cluster" {

 }

+resource "linode_lke_cluster" "prd-video-cluster" {
+  label       = "prd-video-cluster"
+  k8s_version = "1.21"
+  region      = "us-central"
+  tags        = ["production"]
+
+  pool {
+    type  = "g6-standard-2"
+    count = 3
+  }
+
+  # WebRTC-BE pool
+  pool {
+    type  = "g6-standard-2"
+    count = 3
+  }
+  # Coturn pool
+  pool {
+    type  = "g6-standard-2"
+    count = 3
+  }
+
+}
+