Merge 0.7

jieguangzhou · jieguangzhou · commit 40974fb42ab0 · 2025-06-05T15:42:27.000+08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,13 +24,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Handle deletion in progress with status updates
 - Add postprocess in Template
 - Remove the dependencies property from `Component`
+- Fix the bug where the parent of dependent_tables was incorrect when deleting a component.
 
 ### New features
 
 - Add pydantic schema support
 - Add recursive error propagation
 - Add reconnection support for Snowflake vector search
-- Add scanning for leaked passwords on the CI 
+- Add scanning for leaked passwords on the CI
 - Simple detached-lifetime compute and vector-search implementations
 
 ### Bug fixes
diff --git a/plugins/snowflake/superduper_snowflake/__init__.py b/plugins/snowflake/superduper_snowflake/__init__.py
@@ -2,7 +2,8 @@
 from .secrets import check_secret_updates, raise_if_secrets_pending, secrets_not_ready
 from .vector_search import SnowflakeVectorSearcher as VectorSearcher
 
-__version__ = "0.7.0"
+__version__ = "0.7.4"
+
 
 __all__ = [
     "VectorSearcher",
diff --git a/plugins/snowflake/superduper_snowflake/data_backend.py b/plugins/snowflake/superduper_snowflake/data_backend.py
@@ -181,8 +181,9 @@ def drop(self, force: bool = False):
 
     def list_tables(self):
         """List all tables or collections in the database."""
-        results = self.session.sql("SHOW TABLES").collect()
-        return [r.name for r in results]
+        results_tables = self.session.sql("SHOW TABLES").collect()
+        results_views = self.session.sql("SHOW VIEWS").collect()
+        return [r.name for r in results_tables] + [r.name for r in results_views]
 
     ########################################################
     # Abstract methods/ optional methods to be implemented #
diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "superduper-framework"
 description = "Build compositional and declarative AI applications and agents"
 readme = "README.md"
-version = '0.7.0'
+version = '0.7.3'
 license = {file = "LICENSE"}
 maintainers = [{name = "superduper.io, Inc.", email = "opensource@superduper.com"}]
 keywords = [
@@ -70,7 +70,7 @@ test = [
     "pre-commit",
     "black==25.1.0",
     "ruff==0.4.4",
-    "mypy",
+    "mypy==1.15.0",
     "types-PyYAML",
     "types-requests",
     "interrogate",
diff --git a/superduper/base/base.py b/superduper/base/base.py
@@ -272,7 +272,22 @@ def encode(
         for k, v in kwargs.items():
             setattr(context, k, v)
 
-        r = self.dict()
+        if context.keep_variables:
+            r = self._original_parameters
+        else:
+            r = self.dict()
+
+        if not context.include_defaults:
+            for k, v in list(r.items()):
+                if not v:
+                    del r[k]
+            if 'details' in r:
+                del r['details']
+            if 'status' in r:
+                del r['status']
+            if 'version' in r:
+                del r['version']
+
         r = self.class_schema.encode_data(r, context=context)
 
         def _replace_loads_with_references(record, lookup):
diff --git a/superduper/base/datalayer.py b/superduper/base/datalayer.py
@@ -486,11 +486,14 @@ def _build_remove(
             )
         )
         for table in object.dependent_tables:
+            table_parents = self.metadata.get_component_parents(
+                component='Table', identifier=table
+            )
             events.append(
                 Delete(
                     component='Table',
                     identifier=table,
-                    parents=[':'.join(p) for p in parents],
+                    parents=[':'.join(p) for p in table_parents],
                 )
             )
 
diff --git a/superduper/base/datatype.py b/superduper/base/datatype.py
@@ -121,7 +121,24 @@ def encode_data(self, item, context):
             context.builds[key] = item
             return '?' + key
 
-        r = item.dict()
+        if context.keep_variables:
+            r = item._original_parameters
+        else:
+            r = item.dict()
+
+        if not context.include_defaults:
+            from superduper.base.document import Document
+
+            for k, v in list(r.items()):
+                if not v:
+                    del r[k]
+            if 'status' in r:
+                del r['status']
+            if 'details' in r:
+                del r['details']
+            if 'version' in r:
+                del r['version']
+
         if r.schema:
             r = dict(r.schema.encode_data(r, context))
         else:
diff --git a/superduper/base/encoding.py b/superduper/base/encoding.py
@@ -22,6 +22,8 @@ class EncodeContext:
     :param metadata: Whether to include metadata.
     :param defaults: Whether to include defaults.
     :param cfg: Configuration object.
+    :param keep_variables: Whether to keep variables.
+    :param include_defaults: Whether to include default values.
     """
 
     name: str = '__main__'
@@ -33,6 +35,8 @@ class EncodeContext:
     metadata: bool = True
     defaults: bool = True
     cfg: t.Optional[Config] = None
+    keep_variables: bool = False
+    include_defaults: bool = True
 
     def __call__(self, name: str):
         return EncodeContext(
@@ -44,5 +48,7 @@ def __call__(self, name: str):
             leaves_to_keep=self.leaves_to_keep,
             metadata=self.metadata,
             defaults=self.defaults,
+            keep_variables=self.keep_variables,
+            include_defaults=self.include_defaults,
             cfg=self.cfg,
         )
diff --git a/superduper/base/metadata.py b/superduper/base/metadata.py
@@ -227,6 +227,9 @@ def set_failed(self, db: 'Datalayer', reason: str, message: str | None = None):
         for idep in self.inverse_dependencies:
             logging.info(f'Setting downstream job {idep} status to failed')
             job = db['Job'].get(job_id=idep, decode=True)
+            if job is None:
+                continue
+
             job.set_failed(
                 db, reason=f"Upstream dependency {self.job_id} failed", message=None
             )
diff --git a/superduper/base/schema.py b/superduper/base/schema.py
@@ -192,9 +192,6 @@ def encode_data(self, out, context: t.Optional[EncodeContext] = None, **kwargs):
 
             assert field is not None
 
-            # TODO
-            # field.validate(out[k])
-
             try:
                 encoded = field.encode_data(
                     out[k],
diff --git a/superduper/components/application.py b/superduper/components/application.py
@@ -1,35 +1,54 @@
 import typing as t
+from contextlib import contextmanager
 
 from superduper import CFG, logging
-from superduper.base.annotations import trigger
-from superduper.base.status import STATUS_RUNNING
 
-from .component import Component
+from .component import Component, build_vars_var
 
 if t.TYPE_CHECKING:
     from superduper.base.datalayer import Datalayer
 
 
+# 3. The context-manager that temporarily sets the variable
+@contextmanager
+def build_context(vars_dict: dict[str, t.Any] | None):
+    """Context manager to set build variables for components.
+
+    :param vars_dict: Dictionary of variables to set for the build context.
+    """
+    token = build_vars_var.set(vars_dict or {})
+    try:
+        yield
+    finally:
+        build_vars_var.reset(token)
+
+
 class Application(Component):
     """
     A placeholder to hold list of components with associated funcionality.
 
     Components are sorted in a way that respects their mutual dependencies.
 
     :param components: List of components to group together and apply to `superduper`.
-    :param link: A reference link to web app serving the application
-                 i.e. streamlit, gradio, etc
     :param build_variables: Variables which were supplied to a template to build.
     :param build_template: Template which was used to build.
+    :param variables: Variables which are used inside the application.
     """
 
     breaks: t.ClassVar[t.Sequence[str]] = ('components',)
     component_cache: t.ClassVar[bool] = True
 
     components: t.List[Component]
-    link: t.Optional[str] = None
     build_variables: t.Dict | None = None
     build_template: str | None = None
+    variables: t.Dict | None = None
+
+    def postinit(self):
+        """Post-initialization method to set up the application."""
+        with build_context(self.variables):
+            for component in self.components:
+                component.postinit()
+            return super().postinit()
 
     @classmethod
     def build_from_db(cls, identifier, db: "Datalayer"):
diff --git a/superduper/components/component.py b/superduper/components/component.py
@@ -1,11 +1,13 @@
 """The component module provides the base class for all components in superduper.io."""
 
+import contextvars
 import dataclasses as dc
 import json
 import os
 import shutil
 import typing as t
 from collections import OrderedDict, defaultdict
+from contextlib import contextmanager
 from functools import wraps
 
 import networkx
@@ -19,6 +21,7 @@
     STATUS_RUNNING,
     init_status,
 )
+from superduper.base.variables import _replace_variables
 from superduper.misc.annotations import lazy_classproperty
 from superduper.misc.importing import isreallyinstance
 from superduper.misc.utils import hash_item
@@ -113,6 +116,22 @@ def __new__(cls, name, bases, dct):
         return new_cls
 
 
+build_vars_var: contextvars.ContextVar[dict[str, t.Any]] = contextvars.ContextVar(
+    "build_vars_var"
+)
+
+
+def current_build_vars(default: t.Any | None = None) -> dict[str, t.Any] | None:
+    """Get the current build variables.
+
+    :param default: Default value to return if no variables are set.
+    """
+    try:
+        return build_vars_var.get()
+    except LookupError:
+        return default
+
+
 class Component(Base, metaclass=ComponentMeta):
     """Base class for all components in superduper.io.
 
@@ -146,6 +165,7 @@ def __post_init__(self, db: t.Optional['Datalayer'] = None):
         self.db: Datalayer = db
         self.version: t.Optional[int] = None
         self.status, self.details = init_status()
+        self._original_parameters: t.Dict | None = None
         self.postinit()
 
     @property
@@ -507,6 +527,18 @@ def postinit(self):
         """Post initialization method."""
         if not self.identifier:
             raise ValueError('identifier cannot be empty or None')
+        if not self._original_parameters:
+            self._original_parameters = self.dict()
+
+        variables = build_vars_var.get(None)
+        if variables is None:
+            return
+
+        for f in dc.fields(self):
+            attr = getattr(self, f.name)
+            if isinstance(attr, (str, list, dict)) and '<var:' in str(attr):
+                built = _replace_variables(attr, **variables)
+                setattr(self, f.name, built)
 
     def cleanup(self):
         """Method to clean the component."""
@@ -612,7 +644,12 @@ def export(
         # r = r.encode(defaults=defaults, metadata=metadata)
         from superduper import CFG
 
-        r = self.encode(defaults=defaults, metadata=metadata, cfg=CFG(json_native=True))
+        r = self.encode(
+            defaults=defaults,
+            metadata=metadata,
+            cfg=CFG(json_native=True),
+            include_defaults=False,
+        )
 
         def rewrite_keys(r, keys):
             if isinstance(r, dict):
diff --git a/superduper/misc/retry.py b/superduper/misc/retry.py
@@ -61,7 +61,15 @@ def wrapper(*args, **kwargs):
             while attempt <= retries:
                 try:
                     if attempt >= 1:
-                        load_secrets()
+                        try:
+                            load_secrets()
+                        except FileNotFoundError:
+                            raise RuntimeError(
+                                "A secret was not found and the system attempted to "
+                                "load secrets from the secrets volume. "
+                                "However the secrets volume was not found. "
+                                f"Please ensure the secrets volume is mounted. {s.CFG.secrets_volume}"
+                            )
                     return func(*args, **kwargs)
                 except exception_to_check as e:
                     attempt += 1
diff --git a/test/unittest/component/test_model.py b/test/unittest/component/test_model.py
@@ -157,6 +157,7 @@ def test_pm_predict_with_select_ids(monkeypatch, predict_mixin):
             select_using_ids.assert_called_once_with(ids)
 
 
+@pytest.mark.skip
 def test_model_append_metrics():
     @dc.dataclass
     class _Tmp(ObjectModel): ...
@@ -168,6 +169,7 @@ def fit(self, *args, **kwargs): ...
         'test',
         object=object(),
         validation=Validation('test', key=('x', 'y')),
+        # select
         trainer=MyTrainer('test', key='x', select='1'),
     )
 

Original file line number	Diff line number	Diff line change
`@@ -486,11 +486,14 @@ def _build_remove(`
`486`	`486`	`)`
`487`	`487`	`)`
`488`	`488`	`for table in object.dependent_tables:`
	`489`	`+ table_parents = self.metadata.get_component_parents(`
	`490`	`+ component='Table', identifier=table`
	`491`	`+ )`
`489`	`492`	`events.append(`
`490`	`493`	`Delete(`
`491`	`494`	`component='Table',`
`492`	`495`	`identifier=table,`
`493`		`- parents=[':'.join(p) for p in parents],`
	`496`	`+ parents=[':'.join(p) for p in table_parents],`
`494`	`497`	`)`
`495`	`498`	`)`
`496`	`499`