diff --git a/.github/workflows/tutorials-bengali.yml b/.github/workflows/translated-tutorials.yml similarity index 71% rename from .github/workflows/tutorials-bengali.yml rename to .github/workflows/translated-tutorials.yml index 293f1b0ef5a..116d82788fc 100644 --- a/.github/workflows/tutorials-bengali.yml +++ b/.github/workflows/translated-tutorials.yml @@ -1,15 +1,15 @@ -name: Tutorials (Bengali) +name: Tutorials on: push: branches: - master paths: - - "examples/tutorials/translations/bengali/**" + - "examples/tutorials/translations/**" pull_request: types: [opened, synchronize, reopened] paths: - - "examples/tutorials/translations/bengali/**" + - "examples/tutorials/translations/**" jobs: build: @@ -37,6 +37,7 @@ jobs: pip3 install -r pip-dep/requirements_notebooks.txt pip3 install scikit-learn python setup.py install - - name: Run the translated tutorials + - name: Test with pytest run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_bengali + git diff --name-only origin/${{ github.base_ref }} examples/tutorials/translations/ > ./test/notebooks/git-diff.txt + pytest test/notebooks/test_notebooks.py::test_notebooks_basic_translations_diff diff --git a/.github/workflows/tutorials-chinese.yml b/.github/workflows/tutorials-chinese.yml deleted file mode 100644 index 4976c2c90c6..00000000000 --- a/.github/workflows/tutorials-chinese.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Chinese) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/chinese/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/chinese/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_chinese diff --git "a/.github/workflows/tutorials-espa\303\261ol.yml" "b/.github/workflows/tutorials-espa\303\261ol.yml" deleted file mode 100644 index 7ba02e51859..00000000000 --- "a/.github/workflows/tutorials-espa\303\261ol.yml" +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Español) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/español/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/español/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_español diff --git a/.github/workflows/tutorials-hindi.yml b/.github/workflows/tutorials-hindi.yml deleted file mode 100644 index f3ed1297d3d..00000000000 --- a/.github/workflows/tutorials-hindi.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Hindi) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/hindi/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/hindi/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_hindi diff --git a/.github/workflows/tutorials-indonesian.yml b/.github/workflows/tutorials-indonesian.yml deleted file mode 100644 index 6523cfc14b6..00000000000 --- a/.github/workflows/tutorials-indonesian.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Indonesian) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/indonesian/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/indonesian/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_indonesian diff --git a/.github/workflows/tutorials-korean.yml b/.github/workflows/tutorials-korean.yml deleted file mode 100644 index ed93f7281ab..00000000000 --- a/.github/workflows/tutorials-korean.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Korean) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/korean/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/korean/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_korean diff --git "a/.github/workflows/tutorials-portugu\303\252s.yml" "b/.github/workflows/tutorials-portugu\303\252s.yml" deleted file mode 100644 index d9620e3f6f1..00000000000 --- "a/.github/workflows/tutorials-portugu\303\252s.yml" +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Português) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/português/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/português/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_português diff --git a/.github/workflows/tutorials-romanian.yml b/.github/workflows/tutorials-romanian.yml deleted file mode 100644 index 417693f095d..00000000000 --- a/.github/workflows/tutorials-romanian.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Romanian) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/romanian/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/romanian/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_romanian diff --git a/.github/workflows/tutorials-ukrainian.yml b/.github/workflows/tutorials-ukrainian.yml deleted file mode 100644 index f976f2b4e07..00000000000 --- a/.github/workflows/tutorials-ukrainian.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tutorials (Ukrainian) - -on: - push: - branches: - - master - paths: - - "examples/tutorials/translations/ukrainian/**" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "examples/tutorials/translations/ukrainian/**" - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - hash -r - pip3 install -r pip-dep/requirements.txt - pip3 install -r pip-dep/requirements_dev.txt - pip3 install -r pip-dep/requirements_udacity.txt - pip3 install -r pip-dep/requirements_notebooks.txt - pip3 install scikit-learn - python setup.py install - - name: Run the translated tutorials - run: | - pytest test/notebooks/test_notebooks.py::test_basic_notebook_translations_in_ukrainian diff --git a/test/notebooks/git-diff.txt b/test/notebooks/git-diff.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/notebooks/test_notebooks.py b/test/notebooks/test_notebooks.py index 2ada37e27af..253e1599a51 100644 --- a/test/notebooks/test_notebooks.py +++ b/test/notebooks/test_notebooks.py @@ -5,6 +5,7 @@ import urllib.request from pathlib import Path from zipfile import ZipFile +import codecs import pytest import nbformat @@ -17,8 +18,6 @@ from syft import TorchHook from syft.workers.websocket_server import WebsocketServerWorker -thismodule = sys.modules[__name__] - # lets start by finding all notebooks currently available in examples and subfolders all_notebooks = [n for n in glob.glob("examples/tutorials/**/*.ipynb", recursive=True)] basic_notebooks = [n for n in glob.glob("examples/tutorials/*.ipynb")] @@ -34,6 +33,21 @@ Path(nb).name for part in ["10", "13b", "13c"] for nb in translated_notebooks if part in nb ] + +# Include only the translations that have been changed +gitdiff = Path("test/notebooks/git-diff.txt") +changed_files = [] +if gitdiff.is_file(): + changed_files = open(gitdiff, "r") + changed_files = changed_files.readlines() + changed_files = [ + codecs.decode(file.replace('"', "").replace("\n", ""), "unicode-escape") + .encode("latin-1") + .decode() + for file in changed_files + ] +translated_notebooks_diff = list(set(changed_files) & set(translated_notebooks)) + # buggy notebooks with explanation what does not work exclusion_list_notebooks = [ # Part 10 needs either torch.log2 to be implemented or numpy to be hooked @@ -91,45 +105,48 @@ def test_notebooks_basic(isolated_filesystem, notebook): assert isinstance(res, nbformat.notebooknode.NotebookNode) -for language_dir in glob.glob("examples/tutorials/translations/*"): - language = Path(language_dir).name - language_notebooks = [ - n for n in glob.glob(f"examples/tutorials/translations/{language}/*.ipynb", recursive=True) - ] +@pytest.mark.translation +@pytest.mark.parametrize( + "translated_notebook", sorted(set(translated_notebooks) - set(excluded_notebooks)) +) +def test_notebooks_basic_translations(isolated_filesystem, translated_notebook): # pragma: no cover + """Test Notebooks in the tutorial translations folder.""" + notebook = "/".join(translated_notebook.split("/")[-2:]) + notebook = f"translations/{notebook}" + list_name = Path(f"examples/tutorials/{notebook}") + tested_notebooks.append(str(list_name)) + res = pm.execute_notebook( + notebook, + "/dev/null", + parameters={"epochs": 1, "n_test_batches": 5, "n_train_items": 64, "n_test_items": 64}, + timeout=300, + ) + assert isinstance(res, nbformat.notebooknode.NotebookNode) - def language_test_helper(language): - @pytest.mark.translation - @pytest.mark.parametrize( - "language_notebook", sorted(set(language_notebooks) - set(excluded_notebooks)) - ) - def test_basic_notebook_translations_per_language( - isolated_filesystem, language_notebook - ): # pragma: no cover - """Test Notebooks in the tutorial translations folder.""" - notebook = "/".join(language_notebook.split("/")[-2:]) - notebook = f"translations/{notebook}" - list_name = Path(f"examples/tutorials/{notebook}") - tested_notebooks.append(str(list_name)) - res = pm.execute_notebook( - notebook, - "/dev/null", - parameters={ - "epochs": 1, - "n_test_batches": 5, - "n_train_items": 64, - "n_test_items": 64, - }, - timeout=300, - ) - assert isinstance(res, nbformat.notebooknode.NotebookNode) - - return test_basic_notebook_translations_per_language - - setattr( - thismodule, - f"test_basic_notebook_translations_in_{language}", - language_test_helper(language), + +@pytest.mark.translation +@pytest.mark.parametrize( + "translated_notebook", sorted(set(translated_notebooks_diff) - set(excluded_notebooks)) +) +def test_notebooks_basic_translations_diff( + isolated_filesystem, translated_notebook +): # pragma: no cover + """ + Test Notebooks in the tutorial translations folder if they have been + modified in the current pull request. This test should not consider any + notebooks locally. It should be used on Github Actions. + """ + notebook = "/".join(translated_notebook.split("/")[-2:]) + notebook = f"translations/{notebook}" + list_name = Path(f"examples/tutorials/{notebook}") + tested_notebooks.append(str(list_name)) + res = pm.execute_notebook( + notebook, + "/dev/null", + parameters={"epochs": 1, "n_test_batches": 5, "n_train_items": 64, "n_test_items": 64}, + timeout=300, ) + assert isinstance(res, nbformat.notebooknode.NotebookNode) @pytest.mark.parametrize("notebook", sorted(set(advanced_notebooks) - set(excluded_notebooks))) @@ -206,18 +223,7 @@ def test_fl_with_websockets_and_averaging( sy.VirtualWorker(id=n, hook=hook, is_client_worker=False) -def test_github_workflows_exist_for_all_languages(): - dirs_checked = 0 - # check that all languages in translation directory have a workflow - for language_dir in glob.glob("examples/tutorials/translations/*"): - language = Path(language_dir).name - workflow = Path(".github", "workflows", f"tutorials-{language}.yml") - assert os.path.exists(workflow) and os.path.isfile(workflow) - dirs_checked += 1 - assert dirs_checked > 0 - - -### This test must always be last +### These tests must always be last def test_all_notebooks_except_translations(): untested_notebooks = ( set(all_notebooks) @@ -226,3 +232,9 @@ def test_all_notebooks_except_translations(): - set(tested_notebooks) ) assert len(untested_notebooks) == 0, untested_notebooks + + +@pytest.mark.translation +def test_all_translation_notebooks(): # pragma: no cover + untested_notebooks = set(translated_notebooks) - set(excluded_notebooks) - set(tested_notebooks) + assert len(untested_notebooks) == 0, untested_notebooks