Postgres K8s integration

.github/workflows/container-scan.yml

@@ -224,7 +224,7 @@ jobs:
           name: syft.sbom.json
           path: syft.sbom.json
 
-  scan-mongo-latest-trivy:
+  scan-postgres-latest-trivy:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -238,24 +238,24 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "mongo:7.0.0"
+          image-ref: "postgres:16.1"
           format: "cyclonedx"
-          output: "mongo-trivy-results.sbom.json"
+          output: "postgres-trivy-results.sbom.json"
           timeout: "10m0s"
 
         #Upload SBOM to GitHub Artifacts
       - name: Upload SBOM to GitHub Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: mongo-trivy-results.sbom.json
-          path: mongo-trivy-results.sbom.json
+          name: postgres-trivy-results.sbom.json
+          path: postgres-trivy-results.sbom.json
 
           #Generate sarif file
       - name: Run Trivy vulnerability scanner
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "mongo:7.0.0"
+          image-ref: "postgres:16.1"
           format: "sarif"
           output: "trivy-results.sarif"
           timeout: "10m0s"
@@ -266,7 +266,7 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
-  scan-mongo-latest-snyk:
+  scan-postgres-latest-snyk:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -281,7 +281,7 @@ jobs:
           # This is where you will need to introduce the Snyk API token created with your Snyk account
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: mongo:7.0.0
+          image: postgres:16.1
           args: --sarif-file-output=snyk-code.sarif
 
         # Replace any "undefined" security severity values with 0. The undefined value is used in the case

.isort.cfg

@@ -20,4 +20,4 @@ import_heading_localfolder=relative
 ignore_comments=False
 force_grid_wrap=True
 honor_noqa=True
-skip_glob=packages/syft/src/syft/__init__.py,packages/grid/data/*,packages/syft/tests/mongomock/*
+skip_glob=packages/syft/src/syft/__init__.py,packages/grid/data/*

.pre-commit-config.yaml

@@ -3,41 +3,36 @@ repos:
     rev: v4.5.0
     hooks:
       - id: check-ast
-        exclude: ^(packages/syft/tests/mongomock)
         always_run: true
       - id: trailing-whitespace
         always_run: true
-        exclude: ^(docs/|.+\.md|.bumpversion.cfg|packages/syft/tests/mongomock)
+        exclude: ^(docs/|.+\.md|.bumpversion.cfg)
       - id: check-docstring-first
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: check-json
         always_run: true
-        exclude: ^(packages/grid/frontend/|packages/syft/tests/mongomock|.vscode)
+        exclude: ^(packages/grid/frontend/|.vscode)
       - id: check-added-large-files
         always_run: true
         exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif)
       - id: check-yaml
         always_run: true
-        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/|packages/syft/tests/mongomock)
+        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/)
       - id: check-merge-conflict
         always_run: true
         args: ["--assume-in-merge"]
       - id: check-executables-have-shebangs
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: debug-statements
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: name-tests-test
         always_run: true
-        exclude: ^(.*/tests/utils/)|^(.*fixtures.py|packages/syft/tests/mongomock)|^(tests/scenarios/bigquery/helpers)
+        exclude: ^(.*/tests/utils/)|^(.*fixtures.py)|^(tests/scenarios/bigquery/helpers)
       - id: requirements-txt-fixer
         always_run: true
-        exclude: "packages/syft/tests/mongomock"
       - id: mixed-line-ending
         args: ["--fix=lf"]
-        exclude: '\.bat|\.csv|\.ps1$|packages/syft/tests/mongomock'
+        exclude: '\.bat|\.csv|\.ps1$'
 
   - repo: https://github.com/MarcoGorelli/absolufy-imports # This repository has been archived by the owner on Aug 15, 2023. It is now read-only.
     rev: v0.3.1
@@ -88,7 +83,6 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --show-fixes]
-        exclude: packages/syft/tests/mongomock
         types_or: [python, pyi, jupyter]
       - id: ruff-format
         types_or: [python, pyi, jupyter]
@@ -178,7 +172,7 @@ repos:
     rev: "v3.0.0-alpha.9-for-vscode"
     hooks:
       - id: prettier
-        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|packages/syft/tests/mongomock|.vscode)
+        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|.vscode)
 
   # - repo: meta
   #   hooks:

docs/source/api_reference/syft.store.rst

@@ -32,8 +32,5 @@
    syft.store.kv_document_store
    syft.store.linked_obj
    syft.store.locks
-   syft.store.mongo_client
-   syft.store.mongo_codecs
-   syft.store.mongo_document_store
    syft.store.sqlite_document_store
 

notebooks/tutorials/deployments/03-deploy-k8s-k3d.ipynb

@@ -78,7 +78,7 @@
     "If you want to deploy your Kubernetes cluster in a resource-constrained environment, use the following flags to override the default configurations. Please note that you will need at least 1 CPU and 2 GB of RAM on Docker, and some tests may not work in such low-resource environments:\n",
     "\n",
     "```sh\n",
-    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set server.resourcesPreset=null --set seaweedfs.resourcesPreset=null --set mongo.resourcesPreset=null --set registry.resourcesPreset=null --set proxy.resourcesPreset=null --set frontend.resourcesPreset=null\n",
+    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set server.resourcesPreset=null --set seaweedfs.resourcesPreset=null --set postgres.resourcesPreset=null --set registry.resourcesPreset=null --set proxy.resourcesPreset=null --set frontend.resourcesPreset=null\n",
     "```\n",
     "\n",
     "</div>\n",
@@ -89,7 +89,7 @@
     "If you would like to set your own default password even for the production style deployment, use the following command:\n",
     "\n",
     "```sh\n",
-    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set global.randomizedSecrets=false --set server.secret.defaultRootPassword=\"changethis\" --set seaweedfs.secret.s3RootPassword=\"admin\" --set mongo.secret.rootPassword=\"example\"\n",
+    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set global.randomizedSecrets=false --set server.secret.defaultRootPassword=\"changethis\" --set seaweedfs.secret.s3RootPassword=\"admin\" --set postgres.secret.rootPassword=\"example\"\n",
     "```\n",
     "</div>\n"
    ]

packages/grid/backend/backend.dockerfile

@@ -88,6 +88,6 @@ ENV \
     POSTGRESQL_HOST="localhost" \
     POSTGRESQL_PORT="5432" \
     POSTGRESQL_USERNAME="syft_postgres" \
-    POSTGRESQL_PASSWORD="changethis"
+    POSTGRESQL_PASSWORD="example"
 
 CMD ["bash", "./grid/start.sh"]

packages/grid/backend/grid/core/server.py

@@ -1,3 +1,5 @@
+# stdlib
+
 # syft absolute
 from syft.abstract_server import ServerType
 from syft.server.datasite import Datasite
@@ -14,8 +16,8 @@
 from syft.service.queue.zmq_client import ZMQQueueConfig
 from syft.store.blob_storage.seaweedfs import SeaweedFSClientConfig
 from syft.store.blob_storage.seaweedfs import SeaweedFSConfig
-from syft.store.sqlite_document_store import SQLiteStoreClientConfig
-from syft.store.sqlite_document_store import SQLiteStoreConfig
+from syft.store.db.postgres import PostgresDBConfig
+from syft.store.db.sqlite import SQLiteDBConfig
 from syft.types.uid import UID
 
 # server absolute
@@ -34,12 +36,21 @@ def queue_config() -> ZMQQueueConfig:
     return queue_config
 
 
-def sql_store_config() -> SQLiteStoreConfig:
-    client_config = SQLiteStoreClientConfig(
+def sql_store_config() -> SQLiteDBConfig:
+    return SQLiteDBConfig(
         filename=f"{UID.from_string(get_server_uid_env())}.sqlite",
         path=settings.SQLITE_PATH,
     )
-    return SQLiteStoreConfig(client_config=client_config)
+
+
+def postgresql_store_config() -> PostgresDBConfig:
+    return PostgresDBConfig(
+        host=settings.POSTGRESQL_HOST,
+        port=settings.POSTGRESQL_PORT,
+        user=settings.POSTGRESQL_USERNAME,
+        password=settings.POSTGRESQL_PASSWORD,
+        database=settings.POSTGRESQL_DBNAME,
+    )
 
 
 def seaweedfs_config() -> SeaweedFSConfig:
@@ -74,6 +85,8 @@ def seaweedfs_config() -> SeaweedFSConfig:
 worker_class = worker_classes[server_type]
 
 single_container_mode = settings.SINGLE_CONTAINER_MODE
+db_config = sql_store_config() if single_container_mode else postgresql_store_config()
+
 blob_storage_config = None if single_container_mode else seaweedfs_config()
 queue_config = queue_config()
 
@@ -93,4 +106,5 @@ def seaweedfs_config() -> SeaweedFSConfig:
     smtp_host=settings.SMTP_HOST,
     association_request_auto_approval=settings.ASSOCIATION_REQUEST_AUTO_APPROVAL,
     background_tasks=True,
+    db_config=db_config,
 )

packages/grid/default.env

@@ -77,14 +77,6 @@ KANIKO_VERSION="v1.23.2"
 # Jax
 JAX_ENABLE_X64=True
 
-# Mongo
-MONGO_IMAGE=mongo
-MONGO_VERSION="7.0.8"
-MONGO_HOST=mongo
-MONGO_PORT=27017
-MONGO_USERNAME=root
-MONGO_PASSWORD=example
-
 # Redis
 REDIS_PORT=6379
 REDIS_STORE_DB_ID=0
@@ -119,4 +111,4 @@ POSTGRESQL_DBNAME=syftdb_postgres
 POSTGRESQL_HOST=postgres
 POSTGRESQL_PORT=5432
 POSTGRESQL_USERNAME=syft_postgres
-POSTGRESQL_PASSWORD=changethis
+POSTGRESQL_PASSWORD=example

packages/grid/helm/examples/azure/azure.high.yaml

@@ -38,5 +38,5 @@ registry:
 frontend:
   resourcesPreset: medium
 
-mongo:
+postgres:
   resourcesPreset: large

packages/grid/helm/examples/dev/base.yaml

@@ -21,13 +21,6 @@ server:
   secret:
     defaultRootPassword: changethis
 
-mongo:
-  resourcesPreset: null
-  resources: null
-
-  secret:
-    rootPassword: example
-
 postgres:
   resourcesPreset: null
   resources: null

packages/grid/helm/examples/gcp/gcp.high.yaml

@@ -97,7 +97,7 @@ frontend:
 
 # =================================================================================
 
-mongo:
+postgres:
   resourcesPreset: large
 
 # =================================================================================

packages/grid/helm/examples/gcp/gcp.low.yaml

@@ -97,7 +97,7 @@ frontend:
 
 # =================================================================================
 
-mongo:
+postgres:
   resourcesPreset: large
 
 # =================================================================================

packages/grid/helm/examples/gcp/gcp.nosync.yaml

@@ -67,7 +67,7 @@ frontend:
 
 # =================================================================================
 
-mongo:
+postgres:
   resourcesPreset: large
 
 # =================================================================================