diff --git a/.gitea/workflows/admin-build.yaml b/.gitea/workflows/admin-build.yaml new file mode 100644 index 000000000..f0712cc0f --- /dev/null +++ b/.gitea/workflows/admin-build.yaml @@ -0,0 +1,28 @@ +name: Build Admin +on: + push: + paths: + - 'admin/**' + - '.gitea/workflows/**' + +jobs: + build: + # This label maps to the act-runner we deployed in K8s + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Dagger + run: | + curl -L https://dl.dagger.io/dagger/install.sh | sh + sudo mv bin/dagger /usr/local/bin/ + + - name: Login to Gitea Registry + # Gitea automatically injects GITEA_TOKEN into the runner for registry access + run: echo "${{ secrets.GITEA_TOKEN }}" | docker login git.staging.jamkazam.com -u ${{ gitea.actor }} --password-stdin + + - name: Build and Publish with Dagger + working-directory: ./admin + run: | + dagger call build-local --source=. publish --address=git.staging.jamkazam.com/seth/jam-cloud-admin:latest diff --git a/admin/ci/src/index.ts b/admin/ci/src/index.ts new file mode 100644 index 000000000..d93f2e696 --- /dev/null +++ b/admin/ci/src/index.ts @@ -0,0 +1,77 @@ +import { dag, Secret, Directory, object, func, Platform, Container } from "@dagger.io/dagger" + +@object() +export class Admin { + /** + * Fast Local Validation (ARM64 Native) + */ + @func() + async validate(source: Directory): Promise { + return await this.buildWithNix(source, "linux/arm64"); + } + + /** + * Fast Local Build (ARM64 Native) + */ + @func() + async buildLocal(source: Directory): Promise { + return await this.buildWithNix(source, "linux/arm64"); + } + + /** + * Multi-Arch Production Ship + */ + @func() + async ship(source: Directory, kubeconfig: Secret, registry: string): Promise { + const version = this.generateCalVer(); + const platforms: Platform[] = ["linux/amd64", "linux/arm64"]; + + const builds = platforms.map(async (p) => { + const img = await this.buildWithNix(source, p); + return img.publish(`${registry}/jamkazam-admin:${version}-${p.replace("/", "-")}`); + }); + await Promise.all(builds); + + // K8s Poke & Wait + await dag.container() + .from("bitnami/kubectl") + .withMountedSecret("/root/.kube/config", kubeconfig) + .withExec(["kubectl", "set", "image", "deployment/admin", `web=${registry}/jamkazam-admin:${version}-linux-amd64`]) + .withExec(["kubectl", "rollout", "status", "deployment/admin", "--timeout=120s"]) + .sync(); + + return `Shipped ${version}`; + } + + private async buildWithNix(source: Directory, platform: Platform): Promise { + // 1. Start with a Nix-enabled container + let builder = dag.container({ platform }) + .from("nixpkgs/nix:latest") + .withDirectory("/src", source) + .withWorkdir("/src") + // 2. Build the image tarball via Nix + // Add a dummy env var to bust cache if needed + .withEnvVariable("CACHE_BUST", new Date().getTime().toString()) + .withExec(["nix", "--extra-experimental-features", "nix-command flakes", "build", ".#appImageTarball", "--out-link", "result-tarball"]); + + try { + // Force sync to see build output in terminal + await builder.sync(); + } catch (e) { + // If sync fails, try to get the last bit of stderr to explain why + const stderr = await builder.stderr(); + throw new Error(`Nix build failed: ${e}\nStderr: ${stderr}`); + } + + // 3. Extract the tarball file from the builder container + const tarball = builder.file("result-tarball"); + + // 4. Import the tarball as a new Dagger Container + return dag.container({ platform }).import_(tarball); + } + + private generateCalVer(): string { + const d = new Date(); + return `${d.getFullYear()}.${(d.getMonth()+1)}.${d.getDate()}.${d.getHours()}${d.getMinutes()}`; + } +} diff --git a/admin/flake.nix b/admin/flake.nix new file mode 100644 index 000000000..9ef35dbf5 --- /dev/null +++ b/admin/flake.nix @@ -0,0 +1,134 @@ +{ + description = "SOTA Rails 8 Native Environment & OCI Image for JamKazam Admin - Cache Bust 2"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nix2container.url = "github:nlewo/nix2container"; + }; + + outputs = { self, nixpkgs, nix2container }: + let + supportedSystems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + in { + # Track 1: The Native Inner Loop (for 8GB RAM Macs) + devShells = forAllSystems (system: { + default = let pkgs = import nixpkgs { inherit system; }; in + pkgs.mkShell { + buildInputs = with pkgs; [ + ruby_3_4 + postgresql_16.lib + jemalloc + libyaml + vips + bun # for ci/watch.ts + git + ]; + shellHook = '' + export RUBY_YJIT_ENABLE=1 + export RAILS_ENV=development + # macOS uses .dylib, Linux uses .so + if [ "$(uname)" = "Darwin" ]; then + export LD_PRELOAD="${pkgs.jemalloc}/lib/libjemalloc.dylib" + else + export LD_PRELOAD="${pkgs.jemalloc}/lib/libjemalloc.so" + fi + echo "๐Ÿš€ SOTA Shell Ready (YJIT + jemalloc enabled)" + ''; + }; + }); + + # Track 2: The Production OCI Image + packages = forAllSystems (system: + let + pkgs = import nixpkgs { inherit system; }; + n2c = nix2container.packages.${system}; + + # 1. Define the Ruby Environment with common native gems pre-installed + rubyEnv = pkgs.ruby_3_4.withPackages (ps: with ps; [ + psych + pg + nokogiri + ]); + + # 2. Join dev and lib outputs for common dependencies + libyaml-joined = pkgs.symlinkJoin { name = "libyaml-joined"; paths = [ pkgs.libyaml pkgs.libyaml.dev ]; }; + openssl-joined = pkgs.symlinkJoin { name = "openssl-joined"; paths = [ pkgs.openssl pkgs.openssl.dev ]; }; + postgres-joined = pkgs.symlinkJoin { name = "postgres-joined"; paths = [ pkgs.postgresql_16 pkgs.postgresql_16.lib ]; }; + + # 3. Aggregate all dependencies into a single environment + allDeps = pkgs.buildEnv { + name = "jam-admin-env"; + paths = with pkgs; [ + rubyEnv + postgres-joined + libyaml-joined + openssl-joined + zlib.dev + zlib + libiconv + libxml2 + libxml2.dev + libxslt + libxslt.dev + jemalloc + vips + bash + coreutils + git + gnumake + gcc + pkg-config + binutils + ]; + }; + + # 4. A startup script to ensure we are in the right place + start-jam-admin = pkgs.writeShellScriptBin "start-jam-admin" '' + export PATH=${allDeps}/bin:$PATH + export LD_LIBRARY_PATH=${allDeps}/lib:$LD_LIBRARY_PATH + export PKG_CONFIG_PATH=${allDeps}/lib/pkgconfig + + cd /jam-cloud/admin + + echo "๐Ÿ”ง Configuring bundler for native extensions..." + bundle config build.psych --with-yaml-dir=${libyaml-joined} + bundle config build.pg --with-pg-config=${postgres-joined}/bin/pg_config + + echo "๐Ÿ’Ž Installing gems..." + bundle install + + echo "๐Ÿš€ Starting Rails..." + exec bundle exec rails server -b 0.0.0.0 + ''; + + appImage = pkgs.dockerTools.buildLayeredImage { + name = "jamkazam-admin-v3"; + tag = "local"; + config = { + Cmd = [ "${start-jam-admin}/bin/start-jam-admin" ]; + Env = [ + "RUBY_YJIT_ENABLE=1" + "LD_PRELOAD=${pkgs.jemalloc}/lib/libjemalloc.so" + "RAILS_ENV=production" + "RAILS_SERVE_STATIC_FILES=true" + "PKG_CONFIG_PATH=${allDeps}/lib/pkgconfig" + ]; + ExposedPorts = { "3000/tcp" = {}; }; + }; + contents = [ + allDeps + start-jam-admin + pkgs.libyaml.dev + pkgs.openssl.dev + pkgs.postgresql_16.lib + pkgs.zlib.dev + ]; + }; + in { + inherit appImage; + appImageTarball = appImage; + } + ); + }; +} diff --git a/admin/justfile b/admin/justfile new file mode 100644 index 000000000..e291af5a3 --- /dev/null +++ b/admin/justfile @@ -0,0 +1,73 @@ +set shell := ["bash", "-c"] + +NIX := "/nix/var/nix/profiles/default/bin/nix" + +# Start local backing services (one-time, no boot persistence) +infra: + @echo "๐Ÿฐ Starting RabbitMQ..." + brew services run rabbitmq || true + @echo "๐Ÿ’พ Starting Redis..." + brew services run redis || true + @echo "โœ… Local infra is running. Use 'brew services list' to verify." + +# Setup environment +setup: + @echo "๐Ÿ”ง Installing SOTA stack dependencies..." + cd ci && ~/.bun/bin/bun install + @if [ ! -f dagger.json ]; then \ + echo "Initializing Dagger module..."; \ + dagger init --sdk=typescript --source=ci; \ + fi + +# Start coding natively (Track 1) +watch: + ~/.bun/bin/bun run ci/watch.ts + +# Enter the native Nix development environment +shell: + {{NIX}} develop + +# Fast native validation in Dagger (Track 2) +validate: + dagger call --progress=plain validate --source=. + +# Build local ARM64 image via Dagger and export to OrbStack +build: + @echo "๐Ÿ”จ Building image via Dagger..." + @rm -f ./admin.tar + dagger call --progress=plain build-local --source=. export --path=./admin.tar + @echo "๐Ÿงน Cleaning up old Docker artifacts to free space..." + docker system prune -f + @echo "๐Ÿ“ฆ Loading image into Docker..." + #!/bin/bash + LOAD_OUTPUT=$(docker load < ./admin.tar) + echo "$LOAD_OUTPUT" + IMAGE_ID=$(echo "$LOAD_OUTPUT" | awk '/Loaded image ID: sha256:/ {print $4}' | cut -d: -f2) + if [ -z "$IMAGE_ID" ]; then + IMAGE_ID=$(echo "$LOAD_OUTPUT" | grep -oE '[0-9a-f]{12,}' | tail -n 1) + fi + echo "Tagging $IMAGE_ID as jamkazam-admin:local-v2..." + docker tag "$IMAGE_ID" jamkazam-admin:local-v2 + rm ./admin.tar + echo "โœ… Image loaded as 'jamkazam-admin:local-v2'" + +# Run Rails server natively using the Nix shell (SOTA Inner Loop) +dev: infra + {{NIX}} develop --command bash -c "bundle install && bundle exec rails server" + +# Run the local image in OrbStack, connected to host infra +run: + docker run -it --rm \ + -p 3000:3000 \ + -v {{invocation_directory()}}/..:/jam-cloud \ + -w /jam-cloud/admin \ + -e DATABASE_URL="postgres://postgres:postgres@host.orb.internal:5432/jam" \ + -e REDIS_URL="redis://host.orb.internal:6379/1" \ + -e RABBITMQ_URL="amqp://guest:guest@host.orb.internal:5672" \ + -e RAILS_MASTER_KEY=$RAILS_MASTER_KEY \ + jamkazam-admin:local-v2 + +# The Big Red Button +ship: + @echo "๐Ÿš€ Shipping to Production..." + dagger call ship --source=. --kubeconfig=file:$KUBECONFIG --registry="your-registry.io" diff --git a/web/flake.nix b/web/flake.nix new file mode 100644 index 000000000..e78267c1c --- /dev/null +++ b/web/flake.nix @@ -0,0 +1,135 @@ +{ + description = "SOTA Rails 8 Native Environment & OCI Image for JamKazam Web"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = { self, nixpkgs }: + let + supportedSystems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + in { + # Track 1: The Native Inner Loop (for 8GB RAM Macs) + devShells = forAllSystems (system: { + default = let pkgs = import nixpkgs { inherit system; }; in + pkgs.mkShell { + buildInputs = with pkgs; [ + ruby_3_4 + postgresql_16.lib + jemalloc + libyaml + vips + bun + git + libxml2 + libxslt + pkg-config + ]; + shellHook = '' + export RUBY_YJIT_ENABLE=1 + export RAILS_ENV=development + # macOS uses .dylib, Linux uses .so + if [ "$(uname)" = "Darwin" ]; then + export LD_PRELOAD="${pkgs.jemalloc}/lib/libjemalloc.dylib" + else + export LD_PRELOAD="${pkgs.jemalloc}/lib/libjemalloc.so" + fi + echo "๐Ÿš€ SOTA Web Shell Ready (YJIT + jemalloc enabled)" + ''; + }; + }); + + # Track 2: The Production OCI Image + packages = forAllSystems (system: + let + pkgs = import nixpkgs { inherit system; }; + + # 1. Define the Ruby Environment with common native gems pre-installed + rubyEnv = pkgs.ruby_3_4.withPackages (ps: with ps; [ + psych + pg + nokogiri + ]); + + # 2. Join dev and lib outputs for common dependencies + libyaml-joined = pkgs.symlinkJoin { name = "libyaml-joined"; paths = [ pkgs.libyaml pkgs.libyaml.dev ]; }; + openssl-joined = pkgs.symlinkJoin { name = "openssl-joined"; paths = [ pkgs.openssl pkgs.openssl.dev ]; }; + postgres-joined = pkgs.symlinkJoin { name = "postgres-joined"; paths = [ pkgs.postgresql_16 pkgs.postgresql_16.lib ]; }; + + # 3. Aggregate all dependencies into a single environment + allDeps = pkgs.buildEnv { + name = "jam-web-env"; + paths = with pkgs; [ + rubyEnv + postgres-joined + libyaml-joined + openssl-joined + zlib.dev + zlib + libiconv + libxml2 + libxml2.dev + libxslt + libxslt.dev + jemalloc + vips + bash + coreutils + git + gnumake + gcc + pkg-config + binutils + ]; + }; + + # 4. A startup script + start-jam-web = pkgs.writeShellScriptBin "start-jam-web" '' + export PATH=${allDeps}/bin:$PATH + export LD_LIBRARY_PATH=${allDeps}/lib:$LD_LIBRARY_PATH + export PKG_CONFIG_PATH=${allDeps}/lib/pkgconfig + + cd /jam-cloud/web + + echo "๐Ÿ”ง Configuring bundler for native extensions..." + bundle config build.psych --with-yaml-dir=${libyaml-joined} + bundle config build.pg --with-pg-config=${postgres-joined}/bin/pg_config + + echo "๐Ÿ’Ž Installing gems..." + bundle install + + echo "๐Ÿš€ Starting Rails..." + exec bundle exec rails server -b 0.0.0.0 + ''; + + appImage = pkgs.dockerTools.buildLayeredImage { + name = "jamkazam-web"; + tag = "local"; + config = { + Cmd = [ "${start-jam-web}/bin/start-jam-web" ]; + Env = [ + "RUBY_YJIT_ENABLE=1" + "LD_PRELOAD=${pkgs.jemalloc}/lib/libjemalloc.so" + "RAILS_ENV=production" + "RAILS_SERVE_STATIC_FILES=true" + "PKG_CONFIG_PATH=${allDeps}/lib/pkgconfig" + ]; + ExposedPorts = { "3000/tcp" = {}; }; + }; + contents = [ + allDeps + start-jam-web + pkgs.libyaml.dev + pkgs.openssl.dev + pkgs.postgresql_16.lib + pkgs.zlib.dev + ]; + }; + in { + inherit appImage; + appImageTarball = appImage; + } + ); + }; +} diff --git a/web/justfile b/web/justfile new file mode 100644 index 000000000..5acf22445 --- /dev/null +++ b/web/justfile @@ -0,0 +1,52 @@ +set shell := ["bash", "-c"] + +NIX := "/nix/var/nix/profiles/default/bin/nix" + +# Start local backing services +infra: + @echo "๐Ÿฐ Starting RabbitMQ..." + brew services run rabbitmq || true + @echo "๐Ÿ’พ Starting Redis..." + brew services run redis || true + @echo "โœ… Local infra is running." + +# Setup environment +setup: + @echo "๐Ÿ”ง Installing dependencies..." + # bun install etc + +# Start Rails server natively (Track 1) +dev: infra + {{NIX}} develop --command bash -c "bundle install && bundle exec rails server" + +# Enter the native Nix development environment +shell: + {{NIX}} develop + +# Build local ARM64 image +build: + @echo "๐Ÿ”จ Building image via Nix..." + {{NIX}} build .#appImageTarball --out-link result-tarball + @echo "๐Ÿ“ฆ Loading image into Docker..." + #!/bin/bash + LOAD_OUTPUT=$(docker load < ./result-tarball) + echo "$LOAD_OUTPUT" + IMAGE_ID=$(echo "$LOAD_OUTPUT" | awk '/Loaded image ID: sha256:/ {print $4}' | cut -d: -f2) + if [ -z "$IMAGE_ID" ]; then + IMAGE_ID=$(echo "$LOAD_OUTPUT" | grep -oE '[0-9a-f]{12,}' | tail -n 1) + fi + echo "Tagging $IMAGE_ID as jamkazam-web:local..." + docker tag "$IMAGE_ID" jamkazam-web:local + echo "โœ… Image loaded as 'jamkazam-web:local'" + +# Run the local image +run: + docker run -it --rm \ + -p 3001:3000 \ + -v {{invocation_directory()}}/..:/jam-cloud \ + -w /jam-cloud/web \ + -e DATABASE_URL="postgres://postgres:postgres@host.orb.internal:5432/jam" \ + -e REDIS_URL="redis://host.orb.internal:6379/1" \ + -e RABBITMQ_URL="amqp://guest:guest@host.orb.internal:5672" \ + -e RAILS_MASTER_KEY=$RAILS_MASTER_KEY \ + jamkazam-web:local diff --git a/websocket-gateway/flake.nix b/websocket-gateway/flake.nix new file mode 100644 index 000000000..fef378cad --- /dev/null +++ b/websocket-gateway/flake.nix @@ -0,0 +1,126 @@ +{ + description = "SOTA Rails 8 Native Environment & OCI Image for JamKazam WebSocket Gateway"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = { self, nixpkgs }: + let + supportedSystems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + in { + devShells = forAllSystems (system: { + default = let pkgs = import nixpkgs { inherit system; }; in + pkgs.mkShell { + buildInputs = with pkgs; [ + ruby_3_4 + postgresql_16.lib + jemalloc + libyaml + vips + bun + git + libxml2 + libxslt + pkg-config + ]; + shellHook = '' + export RUBY_YJIT_ENABLE=1 + export RAILS_ENV=development + if [ "$(uname)" = "Darwin" ]; then + export LD_PRELOAD="${pkgs.jemalloc}/lib/libjemalloc.dylib" + else + export LD_PRELOAD="${pkgs.jemalloc}/lib/libjemalloc.so" + fi + echo "๐Ÿš€ SOTA WebSocket Shell Ready" + ''; + }; + }); + + packages = forAllSystems (system: + let + pkgs = import nixpkgs { inherit system; }; + + rubyEnv = pkgs.ruby_3_4.withPackages (ps: with ps; [ + psych + pg + nokogiri + ]); + + libyaml-joined = pkgs.symlinkJoin { name = "libyaml-joined"; paths = [ pkgs.libyaml pkgs.libyaml.dev ]; }; + openssl-joined = pkgs.symlinkJoin { name = "openssl-joined"; paths = [ pkgs.openssl pkgs.openssl.dev ]; }; + postgres-joined = pkgs.symlinkJoin { name = "postgres-joined"; paths = [ pkgs.postgresql_16 pkgs.postgresql_16.lib ]; }; + + allDeps = pkgs.buildEnv { + name = "jam-ws-env"; + paths = with pkgs; [ + rubyEnv + postgres-joined + libyaml-joined + openssl-joined + zlib.dev + zlib + libiconv + libxml2 + libxml2.dev + libxslt + libxslt.dev + jemalloc + vips + bash + coreutils + git + gnumake + gcc + pkg-config + ]; + }; + + start-jam-ws = pkgs.writeShellScriptBin "start-jam-ws" '' + export PATH=${allDeps}/bin:$PATH + export LD_LIBRARY_PATH=${allDeps}/lib:$LD_LIBRARY_PATH + export PKG_CONFIG_PATH=${allDeps}/lib/pkgconfig + + cd /jam-cloud/websocket-gateway + + echo "๐Ÿ”ง Configuring bundler..." + bundle config build.psych --with-yaml-dir=${libyaml-joined} + bundle config build.pg --with-pg-config=${postgres-joined}/bin/pg_config + + echo "๐Ÿ’Ž Installing gems..." + bundle install + + echo "๐Ÿš€ Starting WebSocket Gateway..." + # TODO: Add actual start command if different from Rails + exec bundle exec ruby bin/server + ''; + + appImage = pkgs.dockerTools.buildLayeredImage { + name = "jamkazam-ws"; + tag = "local"; + config = { + Cmd = [ "${start-jam-ws}/bin/start-jam-ws" ]; + Env = [ + "RUBY_YJIT_ENABLE=1" + "LD_PRELOAD=${pkgs.jemalloc}/lib/libjemalloc.so" + "RAILS_ENV=production" + "PKG_CONFIG_PATH=${allDeps}/lib/pkgconfig" + ]; + }; + contents = [ + allDeps + start-jam-ws + pkgs.libyaml.dev + pkgs.openssl.dev + pkgs.postgresql_16.lib + pkgs.zlib.dev + ]; + }; + in { + inherit appImage; + appImageTarball = appImage; + } + ); + }; +} diff --git a/websocket-gateway/justfile b/websocket-gateway/justfile new file mode 100644 index 000000000..0df6787f9 --- /dev/null +++ b/websocket-gateway/justfile @@ -0,0 +1,35 @@ +set shell := ["bash", "-c"] + +NIX := "/nix/var/nix/profiles/default/bin/nix" + +# Start local backing services +infra: + @echo "๐Ÿฐ Starting RabbitMQ..." + brew services run rabbitmq || true + @echo "๐Ÿ’พ Starting Redis..." + brew services run redis || true + @echo "โœ… Local infra is running." + +# Start server natively +dev: infra + {{NIX}} develop --command bash -c "bundle install && bundle exec ruby bin/server" + +# Enter shell +shell: + {{NIX}} develop + +# Build image +build: + {{NIX}} build .#appImageTarball --out-link result-tarball + docker load < ./result-tarball + docker tag jamkazam-ws:local jamkazam-ws:latest + +# Run container +run: + docker run -it --rm + -v {{invocation_directory()}}/..:/jam-cloud + -w /jam-cloud/websocket-gateway + -e DATABASE_URL="postgres://postgres:postgres@host.orb.internal:5432/jam" + -e REDIS_URL="redis://host.orb.internal:6379/1" + -e RABBITMQ_URL="amqp://guest:guest@host.orb.internal:5672" + jamkazam-ws:latest