From 6e9a89902a07d52ec931e9c5c8331b05c768dd64 Mon Sep 17 00:00:00 2001 From: Saheed Bolarinwa Date: Fri, 6 Mar 2020 20:36:37 +0100 Subject: [PATCH] Remove ZSTD and all references to it In files: CONTRIBUTING.md, README.md all documentation referring to ZSTD were removed. In file: pip-dep/requirements.txt dependency to ZSTD removed In files: syft/serde/compression.py, test/serde/msgpack/test_msgpack_serde.py all codes referring to ZSTD removed --- CONTRIBUTING.md | 2 +- README.md | 15 +-------------- pip-dep/requirements.txt | 3 +-- syft/serde/compression.py | 19 ------------------- test/serde/msgpack/test_msgpack_serde.py | 16 ++-------------- 5 files changed, 5 insertions(+), 50 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0f251fc86e..1d7b9b6574d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ Example: ```python @pytest.mark.parametrize( - "compress, compressScheme", [(True, "lz4"), (False, "lz4"), (True, "zstd"), (False, "zstd")] + "compress, compressScheme", [(True, "lz4"), (False, "lz4")] ) def test_hooked_tensor(self, compress, compressScheme): TorchHook(torch) diff --git a/README.md b/README.md index 9ea6a289806..e05547951e7 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ PySyft is a Python library for secure and private Deep Learning. PySyft decouples private data from model training, using [Federated Learning](https://ai.googleblog.com/2017/04/federated-learning-collaborative.html), [Differential Privacy](https://en.wikipedia.org/wiki/Differential_privacy), -and Encrypted Computation (like +and Encrypted Computation (like [Multi-Party Computation (MPC)](https://en.wikipedia.org/wiki/Secure_multi-party_computation) and [Homomorphic Encryption (HE)](https://en.wikipedia.org/wiki/Homomorphic_encryption)) within the main Deep Learning frameworks like PyTorch and TensorFlow. Join the movement on @@ -69,19 +69,6 @@ installation error related to PyTorch or TF Encrypted, please open an issue on Github or reach out to `#team_pysyft` in Slack. -If you have an installation error regarding zstd, run this command and then re-try installing syft. - -```bash -pip install --upgrade --force-reinstall zstd -``` -If this still doesn't work, and you happen to be on OSX, make -sure you have [OSX command line tools](https://railsapps.github.io/xcode-command-line-tools.html) installed and try again. - -If this still fails, and you are on a Conda environment. It could be -because conda provides its own compiler and linker tools which might -conflict with your system's. In that case we recommend to use a python venv -and try again. - You can also install PySyft from source on a variety of operating systems by following this [installation guide](https://github.com/OpenMined/PySyft/blob/dev/INSTALLATION.md). ## Run Local Notebook Server diff --git a/pip-dep/requirements.txt b/pip-dep/requirements.txt index 7b991e52876..025cbbdbb2f 100644 --- a/pip-dep/requirements.txt +++ b/pip-dep/requirements.txt @@ -12,5 +12,4 @@ tblib~=1.6.0 torchvision~=0.5.0 torch~=1.4.0 websocket_client~=0.57.0 -websockets~=8.1.0 -zstd~=1.4.4.0 \ No newline at end of file +websockets~=8.1.0 \ No newline at end of file diff --git a/syft/serde/compression.py b/syft/serde/compression.py index b4d7b060570..feb7f8a25bb 100644 --- a/syft/serde/compression.py +++ b/syft/serde/compression.py @@ -7,18 +7,15 @@ from lz4 import ( # noqa: F401 frame, ) # needed as otherwise we will get: module 'lz4' has no attribute 'frame' -import zstd from syft.exceptions import CompressionNotFoundException # COMPRESSION SCHEME INT CODES NO_COMPRESSION = 40 LZ4 = 41 -ZSTD = 42 scheme_to_bytes = { NO_COMPRESSION: NO_COMPRESSION.to_bytes(1, byteorder="big"), LZ4: LZ4.to_bytes(1, byteorder="big"), - ZSTD: ZSTD.to_bytes(1, byteorder="big"), } ## SECTION: chosen Compression Algorithm @@ -48,20 +45,6 @@ def apply_lz4_compression(decompressed_input_bin) -> tuple: return lz4.frame.compress(decompressed_input_bin), LZ4 -def apply_zstd_compression(decompressed_input_bin) -> tuple: - """ - Apply ZSTD compression to the input - - Args: - decompressed_input_bin: the binary to be compressed - - Returns: - a tuple (compressed_result, ZSTD) - """ - - return zstd.compress(decompressed_input_bin), ZSTD - - def apply_no_compression(decompressed_input_bin) -> tuple: """ No compression is applied to the input @@ -118,8 +101,6 @@ def _decompress(binary: bin) -> bin: # 1) Decompress or return the original stream if compress_scheme == LZ4: return lz4.frame.decompress(binary) - elif compress_scheme == ZSTD: - return zstd.decompress(binary) elif compress_scheme == NO_COMPRESSION: return binary else: diff --git a/test/serde/msgpack/test_msgpack_serde.py b/test/serde/msgpack/test_msgpack_serde.py index e1c0edd3a4e..064a93a9908 100644 --- a/test/serde/msgpack/test_msgpack_serde.py +++ b/test/serde/msgpack/test_msgpack_serde.py @@ -416,14 +416,10 @@ def test_ndarray_serde(compress): assert numpy.array_equal(arr, arr_serialized_deserialized) -@pytest.mark.parametrize( - "compress_scheme", [compression.LZ4, compression.ZSTD, compression.NO_COMPRESSION] -) +@pytest.mark.parametrize("compress_scheme", [compression.LZ4, compression.NO_COMPRESSION]) def test_compress_decompress(compress_scheme): if compress_scheme == compression.LZ4: compression._apply_compress_scheme = compression.apply_lz4_compression - elif compress_scheme == compression.ZSTD: - compression._apply_compress_scheme = compression.apply_zstd_compression else: compression._apply_compress_scheme = compression.apply_no_compression @@ -434,14 +430,10 @@ def test_compress_decompress(compress_scheme): assert original == decompressed -@pytest.mark.parametrize( - "compress_scheme", [compression.LZ4, compression.ZSTD, compression.NO_COMPRESSION] -) +@pytest.mark.parametrize("compress_scheme", [compression.LZ4, compression.NO_COMPRESSION]) def test_compressed_serde(compress_scheme): if compress_scheme == compression.LZ4: compression._apply_compress_scheme = compression.apply_lz4_compression - elif compress_scheme == compression.ZSTD: - compression._apply_compress_scheme = compression.apply_zstd_compression else: compression._apply_compress_scheme = compression.apply_no_compression @@ -626,8 +618,6 @@ def test_float(compress): [ (True, compression.LZ4), (False, compression.LZ4), - (True, compression.ZSTD), - (False, compression.ZSTD), (True, compression.NO_COMPRESSION), (False, compression.NO_COMPRESSION), ], @@ -636,8 +626,6 @@ def test_hooked_tensor(compress, compress_scheme): if compress: if compress_scheme == compression.LZ4: compression._apply_compress_scheme = compression.apply_lz4_compression - elif compress_scheme == compression.ZSTD: - compression._apply_compress_scheme = compression.apply_zstd_compression else: compression._apply_compress_scheme = compression.apply_no_compression else: