From d0f63aa3db56c74eaa0716474f0b065a40e9e1de Mon Sep 17 00:00:00 2001 From: ZJaume Date: Wed, 14 Sep 2022 13:35:45 +0000 Subject: [PATCH 1/6] Ability to filter available testsets by language pair Using ``` sacrebleu --list -l en-fr ``` test sets available in English-French are listed. --- README.md | 3 ++- sacrebleu/sacrebleu.py | 8 ++++++-- sacrebleu/utils.py | 14 ++++++++++++++ test/test_api.py | 12 ++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1427ddb7..c8caa2ca 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ following command instead, to perform a full installation with dependencies: # Command-line Usage You can get a list of available test sets with `sacrebleu --list`. Please see [DATASETS.md](DATASETS.md) -for an up-to-date list of supported datasets. +for an up-to-date list of supported datasets. You can also list available test sets for a given language pair +with `sacrebleu --list -l en-fr`. ## Basics diff --git a/sacrebleu/sacrebleu.py b/sacrebleu/sacrebleu.py index 3664ae3b..436d04cb 100755 --- a/sacrebleu/sacrebleu.py +++ b/sacrebleu/sacrebleu.py @@ -42,7 +42,7 @@ from .utils import smart_open, filter_subset, get_langpairs_for_testset, get_available_testsets from .utils import print_test_set, print_subset_results, get_reference_files, download_test_set from .utils import args_to_dict, sanity_check_lengths, print_results_table, print_single_results -from .utils import Color +from .utils import get_available_testsets_for_langpair, Color from . import __version__ as VERSION @@ -248,7 +248,11 @@ def main(): print(f'{pair}: {", ".join(fields)}') else: print('The available test sets are:') - for testset in sorted(get_available_testsets()): + if args.langpair: + testsets = get_available_testsets_for_langpair(args.langpair) + else: + testsets = get_available_testsets() + for testset in sorted(testsets): desc = DATASETS[testset].description.strip() print(f'{testset:<30}: {desc}') sys.exit(0) diff --git a/sacrebleu/utils.py b/sacrebleu/utils.py index a061ed1c..049aa8d5 100644 --- a/sacrebleu/utils.py +++ b/sacrebleu/utils.py @@ -456,6 +456,20 @@ def get_available_testsets() -> List[str]: """Return a list of available test sets.""" return sorted(DATASETS.keys(), reverse=True) +def get_available_testsets_for_langpair(langpair: str) -> List[str]: + """Return a list of available test sets for a given language pair""" + parts = langpair.split('-') + srclang = parts[0] + trglang = parts[1] + + testsets = [] + for dataset in DATASETS.values(): + if f'{srclang}-{trglang}' in dataset.langpairs \ + or f'{trglang}-{srclang}' in dataset.langpairs: + testsets.append(dataset.name) + + return testsets + def get_available_origlangs(test_sets, langpair) -> List[str]: """Return a list of origlang values in according to the raw SGM files.""" diff --git a/test/test_api.py b/test/test_api.py index d571e072..3eb71a4e 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -53,6 +53,18 @@ def test_api_get_available_testsets(): assert "slashdot_" + testset not in available +def test_api_get_available_testsets_for_langpair(): + """ + Loop over the datasets directly, and ensure the API function returns + the test sets found. + """ + available = get_available_testsets_for_langpair('en-it') + assert type(available) is list + assert "wmt19" in available + assert "wmt09" in available + assert "wmt15" not in available + + def test_api_get_langpairs_for_testset(): """ Loop over the datasets directly, and ensure the API function From 9b99ffc0a6d06d29e5d3ff400648d9eaf4405ffc Mon Sep 17 00:00:00 2001 From: ZJaume Date: Wed, 14 Sep 2022 16:00:03 +0000 Subject: [PATCH 2/6] Add langpair to output listing --- sacrebleu/sacrebleu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sacrebleu/sacrebleu.py b/sacrebleu/sacrebleu.py index 436d04cb..a0b8838b 100755 --- a/sacrebleu/sacrebleu.py +++ b/sacrebleu/sacrebleu.py @@ -247,10 +247,11 @@ def main(): fields = DATASETS[args.test_set].fieldnames(pair) print(f'{pair}: {", ".join(fields)}') else: - print('The available test sets are:') if args.langpair: + print(f'The available test sets for {args.langpair} are:') testsets = get_available_testsets_for_langpair(args.langpair) else: + print('The available test sets are:') testsets = get_available_testsets() for testset in sorted(testsets): desc = DATASETS[testset].description.strip() From b375a33ea6f958546b9aa08fc65c685a9f4bd1fd Mon Sep 17 00:00:00 2001 From: ZJaume Date: Fri, 30 Sep 2022 14:54:19 +0000 Subject: [PATCH 3/6] Fix api test filter by langpair --- test/test_api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_api.py b/test/test_api.py index 3eb71a4e..511b4c78 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -13,7 +13,7 @@ import pytest -from sacrebleu.utils import get_available_testsets, get_langpairs_for_testset +from sacrebleu.utils import get_available_testsets, get_available_testsets_for_langpair, get_langpairs_for_testset from sacrebleu.utils import get_source_file, get_reference_files from sacrebleu.dataset import DATASETS @@ -60,10 +60,15 @@ def test_api_get_available_testsets_for_langpair(): """ available = get_available_testsets_for_langpair('en-it') assert type(available) is list - assert "wmt19" in available assert "wmt09" in available assert "wmt15" not in available + available = get_available_testsets_for_langpair('en-fr') + assert type(available) is list + assert "wmt11" in available + assert "mtedx/test" in available + assert "wmt20" not in available + def test_api_get_langpairs_for_testset(): """ From 80d4240f172e808c7b932502d886f0db976004c6 Mon Sep 17 00:00:00 2001 From: Matt Post Date: Mon, 3 Oct 2022 11:26:49 -0400 Subject: [PATCH 4/6] Updated CHANGELOG --- sacrebleu/sacrebleu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sacrebleu/sacrebleu.py b/sacrebleu/sacrebleu.py index a0b8838b..1d7b0e3e 100755 --- a/sacrebleu/sacrebleu.py +++ b/sacrebleu/sacrebleu.py @@ -297,7 +297,7 @@ def main(): sacrelogger.error('I need exactly one of (a) a predefined test set (-t) or (b) a list of references') sys.exit(1) elif args.langpair is None: - sacrelogger.error('I need a language pair (-l).') + sacrelogger.error('I need a language pair (-l). Use --list to see available language pairs for this test set.') sys.exit(1) else: for test_set in args.test_set.split(','): From b961ef307b4dea6f54714972c57e1b1fb12b4bed Mon Sep 17 00:00:00 2001 From: Matt Post Date: Mon, 3 Oct 2022 11:29:57 -0400 Subject: [PATCH 5/6] Remove Python 3.6 from testing, added Python 3.10 --- .github/workflows/check-build.yml | 2 +- CHANGELOG.md | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-build.yml b/.github/workflows/check-build.yml index 6f55a008..84c67cc0 100644 --- a/.github/workflows/check-build.yml +++ b/.github/workflows/check-build.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, 3.10] exclude: - os: windows-latest python-version: '3.6' # test fails due to UTF8 stuff diff --git a/CHANGELOG.md b/CHANGELOG.md index e795897c..23cb908e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,18 @@ # Release Notes -- 2.2.2 (2022-09-29) +- 2.2.2 (2022-XX-XX) Features: - - Added `-tok flores101` and `-tok flores200`, a.k.a. `spbleu`. + - (#203) Added `-tok flores101` and `-tok flores200`, a.k.a. `spbleu`. These are multilingual tokenizations that make use of the multilingual SPM models released by Facebook and described in the following papers: * Flores-101: https://arxiv.org/abs/2106.03193 * Flores-200: https://arxiv.org/abs/2207.04672 + - (#211) You can now list all test sets for a language pair with `--list SRC-TRG`. + Thanks to Jaume Zaragoza (@ZJaume) for adding this feature. + + Changes: + - Removed testing support for Python 3.6 (end-of-lifed: https://endoflife.date/python). - 2.2.1 (2022-09-13) Bugfix: Standard usage was returning (and using) each reference twice. From b7520b82f90ae0794d4e6c5b4a77ee580ee120b3 Mon Sep 17 00:00:00 2001 From: Matt Post Date: Tue, 4 Oct 2022 07:30:54 -0400 Subject: [PATCH 6/6] Python versions need to be quoted? --- .github/workflows/check-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-build.yml b/.github/workflows/check-build.yml index 84c67cc0..8a109b8b 100644 --- a/.github/workflows/check-build.yml +++ b/.github/workflows/check-build.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: ["3.7", "3.8", "3.9", "3.10"] exclude: - os: windows-latest python-version: '3.6' # test fails due to UTF8 stuff