diff --git a/docs/concepts/ui.rst b/docs/concepts/ui.rst index 9f47a796f..adc339227 120000 --- a/docs/concepts/ui.rst +++ b/docs/concepts/ui.rst @@ -1 +1 @@ -../hamilton-ui/ui.rst \ No newline at end of file +../hamilton-ui/ui.rst diff --git a/examples/dask/hello_world/business_logic.py b/examples/dask/hello_world/business_logic.py index b4f559c0d..8255669a3 120000 --- a/examples/dask/hello_world/business_logic.py +++ b/examples/dask/hello_world/business_logic.py @@ -1 +1 @@ -../../hello_world/my_functions.py \ No newline at end of file +../../hello_world/my_functions.py diff --git a/examples/ray/hello_world/business_logic.py b/examples/ray/hello_world/business_logic.py index b4f559c0d..8255669a3 120000 --- a/examples/ray/hello_world/business_logic.py +++ b/examples/ray/hello_world/business_logic.py @@ -1 +1 @@ -../../hello_world/my_functions.py \ No newline at end of file +../../hello_world/my_functions.py diff --git a/examples/spark/pandas_on_spark/business_logic.py b/examples/spark/pandas_on_spark/business_logic.py index b4f559c0d..8255669a3 120000 --- a/examples/spark/pandas_on_spark/business_logic.py +++ b/examples/spark/pandas_on_spark/business_logic.py @@ -1 +1 @@ -../../hello_world/my_functions.py \ No newline at end of file +../../hello_world/my_functions.py diff --git a/examples/spark/scraping_and_chunking b/examples/spark/scraping_and_chunking index 4bf9913cf..acf91713f 120000 --- a/examples/spark/scraping_and_chunking +++ b/examples/spark/scraping_and_chunking @@ -1 +1 @@ -/Users/stefankrawczyk/dagworks/hamilton/examples/LLM_Workflows/scraping_and_chunking/spark \ No newline at end of file +/Users/stefankrawczyk/dagworks/hamilton/examples/LLM_Workflows/scraping_and_chunking/spark diff --git a/hamilton/plugins/.cph/.pandas_extensions.py_c745125af4be0661754c2da2281c6cd1.prob b/hamilton/plugins/.cph/.pandas_extensions.py_c745125af4be0661754c2da2281c6cd1.prob new file mode 100644 index 000000000..92f24d5ed --- /dev/null +++ b/hamilton/plugins/.cph/.pandas_extensions.py_c745125af4be0661754c2da2281c6cd1.prob @@ -0,0 +1 @@ +{"name":"Local: pandas_extensions","url":"d:\\avinash_files\\hamiltoon\\hamilton\\hamilton\\plugins\\pandas_extensions.py","tests":[{"id":1768885818931,"input":"","output":""}],"interactive":false,"memoryLimit":1024,"timeLimit":3000,"srcPath":"d:\\avinash_files\\hamiltoon\\hamilton\\hamilton\\plugins\\pandas_extensions.py","group":"local","local":true} diff --git a/hamilton/plugins/pandas_extensions.py b/hamilton/plugins/pandas_extensions.py index b4ce2786d..dc41f26f2 100644 --- a/hamilton/plugins/pandas_extensions.py +++ b/hamilton/plugins/pandas_extensions.py @@ -24,6 +24,8 @@ from pathlib import Path from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, Union +from packaging.version import Version + try: import pandas as pd except ImportError as e: @@ -171,6 +173,7 @@ def applicable_types(cls) -> Collection[Type]: def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs = {} + pandas_version = Version(pd.__version__) if self.sep is not None: kwargs["sep"] = self.sep if self.delimiter is not None: @@ -205,14 +208,10 @@ def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs["keep_default_na"] = self.keep_default_na if self.na_filter is not None: kwargs["na_filter"] = self.na_filter - if self.verbose is not None: - kwargs["verbose"] = self.verbose if self.skip_blank_lines is not None: kwargs["skip_blank_lines"] = self.skip_blank_lines if self.parse_dates is not None: kwargs["parse_dates"] = self.parse_dates - if self.keep_date_col is not None: - kwargs["keep_date_col"] = self.keep_date_col if self.date_format is not None: kwargs["date_format"] = self.date_format if self.dayfirst is not None: @@ -247,8 +246,6 @@ def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs["dialect"] = self.dialect if self.on_bad_lines is not None: kwargs["on_bad_lines"] = self.on_bad_lines - if self.delim_whitespace is not None: - kwargs["delim_whitespace"] = self.delim_whitespace if self.low_memory is not None: kwargs["low_memory"] = self.low_memory if self.memory_map is not None: @@ -257,8 +254,17 @@ def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs["float_precision"] = self.float_precision if self.storage_options is not None: kwargs["storage_options"] = self.storage_options - if pd.__version__ >= "2.0" and self.dtype_backend is not None: - kwargs["dtype_backend"] = self.dtype_backend + if pandas_version < Version("2.0"): + if self.keep_date_col is not None: + kwargs["keep_date_col"] = self.keep_date_col + if self.verbose is not None: + kwargs["verbose"] = self.verbose + if self.delim_whitespace is not None: + kwargs["delim_whitespace"] = self.delim_whitespace + + if pandas_version >= Version("2.0"): + if self.dtype_backend is not None: + kwargs["dtype_backend"] = self.dtype_backend return kwargs diff --git a/plugin_tests/h_dask/resources b/plugin_tests/h_dask/resources index 1e58ceb6d..50d707597 120000 --- a/plugin_tests/h_dask/resources +++ b/plugin_tests/h_dask/resources @@ -1 +1 @@ -../../tests/resources \ No newline at end of file +../../tests/resources diff --git a/plugin_tests/h_narwhals/resources b/plugin_tests/h_narwhals/resources index 1e58ceb6d..50d707597 120000 --- a/plugin_tests/h_narwhals/resources +++ b/plugin_tests/h_narwhals/resources @@ -1 +1 @@ -../../tests/resources \ No newline at end of file +../../tests/resources diff --git a/plugin_tests/h_ray/resources b/plugin_tests/h_ray/resources index 1e58ceb6d..50d707597 120000 --- a/plugin_tests/h_ray/resources +++ b/plugin_tests/h_ray/resources @@ -1 +1 @@ -../../tests/resources \ No newline at end of file +../../tests/resources diff --git a/plugin_tests/h_spark/resources b/plugin_tests/h_spark/resources index 1e58ceb6d..50d707597 120000 --- a/plugin_tests/h_spark/resources +++ b/plugin_tests/h_spark/resources @@ -1 +1 @@ -../../tests/resources \ No newline at end of file +../../tests/resources diff --git a/ui/backend/hamilton_ui b/ui/backend/hamilton_ui index 13cd1fa7f..254defddb 120000 --- a/ui/backend/hamilton_ui +++ b/ui/backend/hamilton_ui @@ -1 +1 @@ -server \ No newline at end of file +server