CLI: move project context commands to CLI root

CHANGELOG.md

@@ -17,9 +17,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `Dataset` class extended with new operations: `save`, `load`, `export`, `import_from`, `detect`, `run_model` (<https://github.com/openvinotoolkit/datumaro/pull/71>)
 - `Dataset` operations return `Dataset` instances, allowing to chain operations (<https://github.com/openvinotoolkit/datumaro/pull/71>)
 - Allowed importing `Extractor`-only defined formats (in `Project.import_from`, `dataset.import_from` and CLI/`project import`)
+- `datum ...` commands replaced with `datum ...` commands (<https://github.com/openvinotoolkit/datumaro/pull/84>)
 
 ### Deprecated
--
+- `datum ...` CLI context (<https://github.com/openvinotoolkit/datumaro/pull/84>)
 
 ### Removed
 -

README.md

@@ -44,23 +44,23 @@ CVAT annotations                             ---> Publication, statistics etc.
 - Convert only non-`occluded` annotations from a [CVAT](https://github.com/opencv/cvat) project to TFrecord:
   ```bash
   # export Datumaro dataset in CVAT UI, extract somewhere, go to the project dir
-  datum project filter -e '/item/annotation[occluded="False"]' \
+  datum filter -e '/item/annotation[occluded="False"]' \
     --mode items+anno --output-dir not_occluded
-  datum project export --project not_occluded \
+  datum export --project not_occluded \
     --format tf_detection_api -- --save-images
   ```
 
 - Annotate MS COCO dataset, extract image subset, re-annotate it in [CVAT](https://github.com/opencv/cvat), update old dataset:
   ```bash
   # Download COCO dataset http://cocodataset.org/#download
   # Put images to coco/images/ and annotations to coco/annotations/
-  datum project import --format coco --input-path <path/to/coco>
-  datum project export --filter '/image[images_I_dont_like]' --format cvat \
+  datum import --format coco --input-path <path/to/coco>
+  datum export --filter '/image[images_I_dont_like]' --format cvat \
     --output-dir reannotation
   # import dataset and images to CVAT, re-annotate
   # export Datumaro project, extract to 'reannotation-upd'
-  datum project project merge reannotation-upd
-  datum project export --format coco
+  datum merge reannotation-upd
+  datum export --format coco
   ```
 
 - Annotate instance polygons in [CVAT](https://github.com/opencv/cvat), export as masks in COCO:
@@ -72,18 +72,18 @@ CVAT annotations                             ---> Publication, statistics etc.
 - Apply an OpenVINO detection model to some COCO-like dataset,
   then compare annotations with ground truth and visualize in TensorBoard:
   ```bash
-  datum project import --format coco --input-path <path/to/coco>
+  datum import --format coco --input-path <path/to/coco>
   # create model results interpretation script
   datum model add mymodel openvino \
     --weights model.bin --description model.xml \
     --interpretation-script parse_results.py
   datum model run --model mymodel --output-dir mymodel_inference/
-  datum project diff mymodel_inference/ --format tensorboard --output-dir diff
+  datum diff mymodel_inference/ --format tensorboard --output-dir diff
   ```
 
 - Change colors in PASCAL VOC-like `.png` masks:
   ```bash
-  datum project import --format voc --input-path <path/to/voc/dataset>
+  datum import --format voc --input-path <path/to/voc/dataset>
 
   # Create a color map file with desired colors:
   #
@@ -93,7 +93,7 @@ CVAT annotations                             ---> Publication, statistics etc.
   #
   # Save as mycolormap.txt
 
-  datum project export --format voc_segmentation -- --label-map mycolormap.txt
+  datum export --format voc_segmentation -- --label-map mycolormap.txt
   # add "--apply-colormap=0" to save grayscale (indexed) masks
   # check "--help" option for more info
   # use "datum --loglevel debug" for extra conversion info

datumaro/cli/__init__.py

@@ -1,4 +1,4 @@
 
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT

datumaro/cli/__main__.py

@@ -1,5 +1,5 @@
 
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -58,18 +58,25 @@ def make_parser():
     _LogManager._define_loglevel_option(parser)
 
     known_contexts = [
-        ('project', contexts.project, "Actions on projects (datasets)"),
-        ('source', contexts.source, "Actions on data sources"),
-        ('model', contexts.model, "Actions on models"),
+        ('project', contexts.project, "Actions with project (deprecated)"),
+        ('source', contexts.source, "Actions with data sources"),
+        ('model', contexts.model, "Actions with models"),
     ]
     known_commands = [
         ('create', commands.create, "Create project"),
-        ('add', commands.add, "Add source to project"),
-        ('remove', commands.remove, "Remove source from project"),
-        ('export', commands.export, "Export project"),
+        ('import', commands.import_, "Create project from existing dataset"),
+        ('add', commands.add, "Add data source to project"),
+        ('remove', commands.remove, "Remove data source from project"),
+        ('export', commands.export, "Export project in some format"),
+        ('filter', commands.filter, "Filter project"),
+        ('transform', commands.transform, "Transform project"),
+        ('merge', commands.merge, "Merge projects"),
+        ('convert', commands.convert, "Convert dataset into another format"),
+        ('diff', commands.diff, "Compare projects with intersection"),
+        ('ediff', commands.ediff, "Compare projects for equality"),
+        ('stats', commands.stats, "Compute project statistics"),
+        ('info', commands.info, "Print project info"),
         ('explain', commands.explain, "Run Explainable AI algorithm for model"),
-        ('merge', commands.merge, "Merge datasets"),
-        ('convert', commands.convert, "Convert dataset"),
     ]
 
     # Argparse doesn't support subparser groups:

datumaro/cli/commands/__init__.py

@@ -1,6 +1,13 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
-from . import add, create, explain, export, remove, merge, convert
+# pylint: disable=redefined-builtin
+
+from . import (
+    create, add, remove, import_,
+    explain,
+    export, merge, convert, transform, filter,
+    diff, ediff, stats,
+    info
+)

datumaro/cli/commands/add.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2020-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 

datumaro/cli/commands/convert.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -64,7 +63,7 @@ def convert_command(args):
     env = Environment()
 
     try:
-        converter = env.converters.get(args.output_format)
+        converter = env.converters[args.output_format]
     except KeyError:
         raise CliException("Converter for format '%s' is not found" % \
             args.output_format)

datumaro/cli/commands/create.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 

datumaro/cli/commands/diff.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2019-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_diff_parser as build_parser

datumaro/cli/commands/ediff.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2019-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_ediff_parser as build_parser

datumaro/cli/commands/explain.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 

datumaro/cli/commands/export.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 

datumaro/cli/commands/filter.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2020-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_filter_parser as build_parser

datumaro/cli/commands/import_.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2019-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_import_parser as build_parser

datumaro/cli/commands/info.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2019-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_info_parser as build_parser

datumaro/cli/commands/merge.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2020 Intel Corporation
+# Copyright (C) 2020-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 

datumaro/cli/commands/remove.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 

datumaro/cli/commands/stats.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2019-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_stats_parser as build_parser

datumaro/cli/commands/transform.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2020-2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_transform_parser as build_parser

datumaro/cli/contexts/__init__.py

@@ -1,6 +1,6 @@
 
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
-from . import project, source, model, item
+from . import project, source, model

datumaro/cli/contexts/model/__init__.py → datumaro/cli/contexts/model.py

@@ -1,19 +1,16 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
 import argparse
 import logging as log
 import os
 import os.path as osp
-import re
 
-from datumaro.components.dataset import DEFAULT_FORMAT
 from datumaro.components.project import Environment
 
-from ...util import CliException, MultilineFormatter, add_subparser
-from ...util.project import load_project, \
+from ..util import CliException, MultilineFormatter, add_subparser
+from ..util.project import load_project, \
     generate_next_name, generate_next_file_name
 
 

datumaro/cli/contexts/project/__init__.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -87,7 +86,7 @@ def create_command(args):
 def build_import_parser(parser_ctor=argparse.ArgumentParser):
     builtins = sorted(Environment().importers.items)
 
-    parser = parser_ctor(help="Create project from existing dataset",
+    parser = parser_ctor(help="Create project from an existing dataset",
         description="""
             Creates a project from an existing dataset. The source can be:|n
             - a dataset in a supported format (check 'formats' section below)|n
@@ -181,14 +180,14 @@ def import_command(args):
 
         matches = env.detect_dataset(args.source)
         if len(matches) == 0:
-            log.error("Failed to detect dataset format automatically. "
+            log.error("Failed to detect dataset format. "
                 "Try to specify format with '-f/--format' parameter.")
             return 1
         elif len(matches) != 1:
             log.error("Multiple formats match the dataset: %s. "
                 "Try to specify format with '-f/--format' parameter.",
                 ', '.join(matches))
-            return 2
+            return 1
 
         fmt = matches[0]
     elif args.extra_args:
@@ -334,7 +333,6 @@ def export_command(args):
     except KeyError:
         raise CliException("Converter for format '%s' is not found" % \
             args.format)
-
     extra_args = converter.parse_cmdline(args.extra_args)
 
     filter_args = FilterModes.make_filter_args(args.filter_mode)
@@ -348,8 +346,7 @@ def export_command(args):
         dataset = dataset.filter(args.filter, **filter_args)
     dataset.export(format=args.format, save_dir=dst_dir, **extra_args)
 
-    log.info("Project exported to '%s' as '%s'" % \
-        (dst_dir, args.format))
+    log.info("Project exported to '%s' as '%s'" % (dst_dir, args.format))
 
     return 0
 

datumaro/cli/contexts/project/diff.py

@@ -1,5 +1,5 @@
 
-# Copyright (C) 2019-2020 Intel Corporation
+# Copyright (C) 2019-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT