cleanup

Author: codetheweb

.github/actions/docker/action.yaml

@@ -18,10 +18,10 @@ runs:
   using: "composite"
   steps:
     # https://github.com/docker/setup-qemu-action - for multiplatform builds
-    # - name: Set up QEMU
-    #   uses: docker/setup-qemu-action@v2
-    # - name: Set up Docker Buildx
-    #   uses: docker/setup-buildx-action@v3
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
     - name: Log in to the Github Container registry
       uses: docker/[email protected]
       if: ${{ inputs.ghcr-username != '' }}

.github/actions/tilt/action.yaml

@@ -3,20 +3,19 @@ description: "This action starts Tilt services"
 runs:
   using: "composite"
   steps:
-    - name: Collect Workflow Telemetry
-      uses: catchpoint/workflow-telemetry-action@v2
     - name: Start minikube
       uses: medyagh/setup-minikube@latest
       with:
-        driver: none
+        driver: none # uses Docker engine on host instead of Docker-in-Docker
+    # tilt ci can automatically build images while bringing up the cluster. However, this results in flaky runs for our usage so we split up image building and pod deployment into separate steps. See https://github.com/chroma-core/chroma/pull/4720.
     - name: Install Tilt, bake images, pre-pull external images
       shell: bash
       env:
         TILT_VERSION: "0.34.2"
       run: |
-        parallel --line-buffer --keep-order ::: \
+        parallel --tag --linebuffer ::: \
           "bash -c 'curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz | tar -xzv -C /usr/local/bin tilt'" \
-          "docker buildx bake -f docker-bake.hcl --load" \
+          "docker buildx bake -f .github/actions/tilt/docker-bake.hcl --load" \
           "bash .github/actions/tilt/pull_external_images.sh"
     - name: Start Tilt
       shell: bash

docker-bake.hcl renamed to .github/actions/tilt/docker-bake.hcl

@@ -1,3 +1,7 @@
+#!/usr/bin/env bash
+
+# This is a simple script to gather all external images referenced in Kubernetes manifests and then pull them in parallel.
+
 set -euo pipefail
 
 k8s_dir="k8s"
@@ -8,12 +12,15 @@ mapfile -t images < <(
   grep -v '{{' |
   sed -E 's/.*image:[[:space:]]*//' |
   tr -d '"' |
-  grep -vi 'chroma-postgres' |                 # EXCLUDE chroma-postgres
-  grep -vi 'load-service' |                 # EXCLUDE chroma-postgres
+  # chroma-postgres appears to be an external image ref, but it's a custom image.
+  # It's just the base postgres image with a single file copied in (k8s/test/postgres/Dockerfile) so it's ok to build during `tilt ci`.
+  grep -vi 'chroma-postgres' |
+  # The load service appears in k8s/test and is not Helm-templated, so we must exclude it here.
+  grep -vi 'load-service' |
   sort -u
 )
 
-(( ${#images[@]} )) || { echo "No literal images found – nothing to pull."; exit 0; }
+(( ${#images[@]} )) || { echo "No literal images found, nothing to pull."; exit 0; }
 
 # Build a temporary docker-compose file
 tmpfile=$(mktemp)

.github/actions/tilt/pull_external_images.sh

@@ -20,7 +20,8 @@ jobs:
         with:
           dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: ./.github/actions/tilt
+      - name: Start Tilt services
+        uses: ./.github/actions/tilt
       - run: bin/cluster-test.sh bash -c 'cd go && make test'
       - name: Save service logs to artifact
         if: always()

.github/workflows/_go-tests.yml

@@ -154,7 +154,8 @@ jobs:
         with:
           dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: ./.github/actions/tilt
+      - name: Start Tilt services
+        uses: ./.github/actions/tilt
       - name: Test
         run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}" --durations 10'
         shell: bash

.github/workflows/_python-tests.yml

@@ -166,7 +166,6 @@ jobs:
     secrets: inherit
     with:
       property_testing_preset: 'normal'
-      python_versions: '["3.9", "3.10", "3.11", "3.12"]'
 
   python-vulnerability-scan:
     name: Python vulnerability scan
@@ -189,8 +188,8 @@ jobs:
 
   go-tests:
     name: Go tests
-    # needs: change-detection
-    # if: contains(fromJson(needs.change-detection.outputs.tests-to-run), 'go')
+    needs: change-detection
+    if: contains(fromJson(needs.change-detection.outputs.tests-to-run), 'go')
     uses: ./.github/workflows/_go-tests.yml
     secrets: inherit
 

.github/workflows/pr.yml

@@ -1,5 +1,7 @@
 update_settings(max_parallel_updates=6)
 
+# *:ci images are defined in .github/actions/tilt/docker-bake.hcl and used for .github/actions/tilt/action.yaml.
+
 if config.tilt_subcommand == "ci":
   custom_build(
     'chroma-postgres',

Tiltfile

@@ -244,15 +244,6 @@ impl TryFrom<chroma_proto::Collection> for Collection {
     }
 }
 
-// todo test rebuild AC7B900F-8EF4-4035-9B09-80DD3D4D8E35
-// 4EEDDBBB-B8E4-4468-BDFD-B1E6A6EBFEF2
-// 86337F43-A95B-4AF4-B68B-03BD35648AEC
-// 59FD8E66-962A-4AAC-8758-43441A0D8575
-// C22A3DB9-4A83-409D-9A6A-BD54466FC536
-// 01AEFDF3-F405-4A86-BF65-148EE050A2F8
-// 71A45988-1989-464D-9780-ADEBF28402A3
-// DCADEFBC-B601-425F-B115-C353F8EFCDA8
-
 #[derive(Error, Debug)]
 pub enum CollectionToProtoError {
     #[error("Could not serialize config: {0}")]