Test verificatin of staging in the pipeline

bitbucket-pipelines.yml

@@ -170,6 +170,28 @@ pipelines:
               variables:
                 KUBE_CONFIG: $KUBE_CONFIG_STG
                 KUBECTL_COMMAND: '-n coturn-dns rollout status -w deployment/coturn-dns'
+      - step:
+          name: Verify and Test Staging
+          deployment: staging
+          image: node:22
+          script:
+            - apt-get update && apt-get install -y curl jq git
+            - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+            - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+            - echo $KUBE_CONFIG_STG | base64 -d > kubeconfig
+            - export KUBECONFIG=$(pwd)/kubeconfig
+            - npx playwright install-deps
+            - chmod +x scripts/verify-deployment.sh
+            - ./scripts/verify-deployment.sh $BITBUCKET_COMMIT staging
+            - git clone --depth 1 git@bitbucket.org:jamkazam/video-e2e.git
+            - cd video-e2e
+            - npm install
+            - npx playwright install chromium
+            - ./bin/staging-test
+          after-script:
+            - if [ $BITBUCKET_EXIT_CODE -ne 0 ]; then
+                curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"🚨 Pipeline Failed: Staging verification/tests failed for commit $BITBUCKET_COMMIT. <https://bitbucket.org/$BITBUCKET_REPO_FULL_NAME/pipelines/results/$BITBUCKET_BUILD_NUMBER|View Log>\"}" https://hooks.slack.com/services/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
+              fi
   custom:
     build-and-push-coturn-dns:
       - variables:
@@ -189,3 +211,26 @@ pipelines:
             - docker push "gcr.io/${GCLOUD_PROJECT}/coturn-dns:${VERSION}"
           services:
             - docker
+    run-staging-test:
+      - step:
+          name: Verify and Test Staging
+          deployment: staging
+          image: node:22
+          script:
+            - apt-get update && apt-get install -y curl jq git
+            - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+            - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+            - echo $KUBE_CONFIG_STG | base64 -d > kubeconfig
+            - export KUBECONFIG=$(pwd)/kubeconfig
+            - npx playwright install-deps
+            - chmod +x scripts/verify-deployment.sh
+            - ./scripts/verify-deployment.sh $BITBUCKET_COMMIT staging
+            - git clone --depth 1 git@bitbucket.org:jamkazam/video-e2e.git
+            - cd video-e2e
+            - npm install
+            - npx playwright install chromium
+            - ./bin/staging-test
+          after-script:
+            - if [ $BITBUCKET_EXIT_CODE -ne 0 ]; then
+                curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"🚨 Pipeline Failed: Staging verification/tests failed for commit $BITBUCKET_COMMIT. <https://bitbucket.org/$BITBUCKET_REPO_FULL_NAME/pipelines/results/$BITBUCKET_BUILD_NUMBER|View Log>\"}" https://hooks.slack.com/services/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
+              fi

scripts/verify-deployment.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -e
+
+COMMIT_HASH=$1
+ENV=$2 # staging or production
+SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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"
+
+if [ -z "$COMMIT_HASH" ]; then
+  echo "Usage: $0 <commit-hash> <env>"
+  exit 1
+fi
+
+echo "Verifying deployment for commit $COMMIT_HASH in $ENV..."
+
+send_slack_alert() {
+  local message=$1
+  echo "Sending Slack Alert: $message"
+  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" $SLACK_WEBHOOK_URL
+}
+
+check_app_sync() {
+  local app_name=$1
+  local timeout=300 # 5 minutes
+  local start_time=$(date +%s)
+
+  echo "Checking sync status for $app_name..."
+
+  while true; do
+    current_time=$(date +%s)
+    if [ $((current_time - start_time)) -gt $timeout ]; then
+      echo "Timeout waiting for $app_name to sync."
+      send_slack_alert "🚨 Deployment Failed: Timeout waiting for $app_name to sync to commit $COMMIT_HASH"
+      exit 1
+    fi
+
+    app_json=$(kubectl get application $app_name -n argocd -o json 2>/dev/null || echo "{}")
+    
+    if [ "$app_json" == "{}" ]; then
+       echo "Application $app_name not found yet..."
+       sleep 10
+       continue
+    fi
+
+    sync_status=$(echo "$app_json" | jq -r '.status.sync.status')
+    health_status=$(echo "$app_json" | jq -r '.status.health.status')
+    revision=$(echo "$app_json" | jq -r '.status.sync.revision')
+    
+    # Check if revision contains the commit hash
+    match=false
+    if [[ "$revision" == *"$COMMIT_HASH"* ]]; then
+        match=true
+    fi
+
+    echo "App: $app_name | Sync: $sync_status | Health: $health_status | Rev: $revision | Target: $COMMIT_HASH"
+
+    if [ "$match" = true ] && [ "$sync_status" == "Synced" ] && [ "$health_status" == "Healthy" ]; then
+      echo "$app_name is Synced and Healthy at $revision"
+      break
+    fi
+
+    sleep 10
+  done
+}
+
+# Check apps
+check_app_sync "monitoring"
+check_app_sync "webrtc-be"
+check_app_sync "coturn"
+
+echo "All apps synced and healthy."