diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b4f544..a284e21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,4 +61,4 @@ jobs: - name: Pytest shell: bash - run: poetry run pytest -v tests + run: poetry run pytest -v tests/unit diff --git a/poetry.lock b/poetry.lock index d81abce..09ab04b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -101,6 +101,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "execnet" +version = "1.9.0" +description = "execnet: rapid multi-Python deployment" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +testing = ["pre-commit"] + [[package]] name = "iniconfig" version = "1.1.1" @@ -172,6 +183,21 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] dev = ["pre-commit", "tox"] +[[package]] +name = "portalocker" +version = "1.7.1" +description = "Wraps the portalocker recipe for easy usage" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +pywin32 = {version = "!=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +tests = ["pytest (>=4.6.9)", "pytest-cov (>=2.8.1)", "sphinx (>=1.8.5)", "pytest-flake8 (>=1.0.5)"] + [[package]] name = "py" version = "1.10.0" @@ -217,6 +243,50 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-forked" +version = "1.3.0" +description = "run tests in isolated forked subprocesses" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +py = "*" +pytest = ">=3.10" + +[[package]] +name = "pytest-terraform" +version = "0.5.3" +description = "A pytest plugin for using terraform fixtures" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +jmespath = ">=0.10.0,<0.11.0" +portalocker = ">=1.7.0,<2.0.0" +pytest = ">=6.0,<7.0" +pytest-xdist = ">=1.31.0" + +[[package]] +name = "pytest-xdist" +version = "2.4.0" +description = "pytest xdist plugin for distributed testing and loop-on-failing modes" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.0.0" +pytest-forked = "*" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -228,6 +298,14 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" [package.dependencies] six = ">=1.5" +[[package]] +name = "pywin32" +version = "302" +description = "Python for Window Extensions" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "regex" version = "2021.8.3" @@ -290,7 +368,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7ca6fe92d0f2372d7e3cde83c0699c8ca40232f4c9ce53c35a24562f2728e35f" +content-hash = "ecd9cddc477dd8f97d1ed627309dab8709b995a1894cb09c3ce5c1ef302ac765" [metadata.files] appdirs = [ @@ -325,6 +403,10 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +execnet = [ + {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, + {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, +] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, @@ -353,6 +435,10 @@ pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] +portalocker = [ + {file = "portalocker-1.7.1-py2.py3-none-any.whl", hash = "sha256:34cb36c618d88bcd9079beb36dcdc1848a3e3d92ac4eac59055bdeafc39f9d4a"}, + {file = "portalocker-1.7.1.tar.gz", hash = "sha256:6d6f5de5a3e68c4dd65a98ec1babb26d28ccc5e770e07b672d65d5a35e4b2d8a"}, +] py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, @@ -388,10 +474,34 @@ pytest = [ {file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"}, {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"}, ] +pytest-forked = [ + {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, + {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, +] +pytest-terraform = [ + {file = "pytest-terraform-0.5.3.tar.gz", hash = "sha256:7e63de138b44d81807d6c5bae32b060c7ce75255e54d53c27956b7aea7792b1a"}, + {file = "pytest_terraform-0.5.3-py3-none-any.whl", hash = "sha256:b400ae8c097e121d41456e086f76a7b6f5e63b34a7320444b0b2fee1b8bb6499"}, +] +pytest-xdist = [ + {file = "pytest-xdist-2.4.0.tar.gz", hash = "sha256:89b330316f7fc475f999c81b577c2b926c9569f3d397ae432c0c2e2496d61ff9"}, + {file = "pytest_xdist-2.4.0-py3-none-any.whl", hash = "sha256:7b61ebb46997a0820a263553179d6d1e25a8c50d8a8620cd1aa1e20e3be99168"}, +] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] +pywin32 = [ + {file = "pywin32-302-cp310-cp310-win32.whl", hash = "sha256:251b7a9367355ccd1a4cd69cd8dd24bd57b29ad83edb2957cfa30f7ed9941efa"}, + {file = "pywin32-302-cp310-cp310-win_amd64.whl", hash = "sha256:79cf7e6ddaaf1cd47a9e50cc74b5d770801a9db6594464137b1b86aa91edafcc"}, + {file = "pywin32-302-cp36-cp36m-win32.whl", hash = "sha256:fe21c2fb332d03dac29de070f191bdbf14095167f8f2165fdc57db59b1ecc006"}, + {file = "pywin32-302-cp36-cp36m-win_amd64.whl", hash = "sha256:d3761ab4e8c5c2dbc156e2c9ccf38dd51f936dc77e58deb940ffbc4b82a30528"}, + {file = "pywin32-302-cp37-cp37m-win32.whl", hash = "sha256:48dd4e348f1ee9538dd4440bf201ea8c110ea6d9f3a5010d79452e9fa80480d9"}, + {file = "pywin32-302-cp37-cp37m-win_amd64.whl", hash = "sha256:496df89f10c054c9285cc99f9d509e243f4e14ec8dfc6d78c9f0bf147a893ab1"}, + {file = "pywin32-302-cp38-cp38-win32.whl", hash = "sha256:e372e477d938a49266136bff78279ed14445e00718b6c75543334351bf535259"}, + {file = "pywin32-302-cp38-cp38-win_amd64.whl", hash = "sha256:543552e66936378bd2d673c5a0a3d9903dba0b0a87235ef0c584f058ceef5872"}, + {file = "pywin32-302-cp39-cp39-win32.whl", hash = "sha256:2393c1a40dc4497fd6161b76801b8acd727c5610167762b7c3e9fd058ef4a6ab"}, + {file = "pywin32-302-cp39-cp39-win_amd64.whl", hash = "sha256:af5aea18167a31efcacc9f98a2ca932c6b6a6d91ebe31f007509e293dea12580"}, +] regex = [ {file = "regex-2021.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8764a78c5464ac6bde91a8c87dd718c27c1cabb7ed2b4beaf36d3e8e390567f9"}, {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4551728b767f35f86b8e5ec19a363df87450c7376d7419c3cac5b9ceb4bce576"}, diff --git a/pyproject.toml b/pyproject.toml index 4f10cf5..5f01910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ jsonschema = "^3.2.0" [tool.poetry.dev-dependencies] pytest = "^6.2.4" black = "^21.7b0" +pytest-terraform = { git = "https://github.com/cloud-custodian/pytest-terraform.git", branch = "work-dir-test-api" } [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/unit/conftest.py b/tests/conftest.py similarity index 78% rename from tests/unit/conftest.py rename to tests/conftest.py index b99838f..0e5d09d 100644 --- a/tests/unit/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,20 @@ import pytest +def load_data(filename, location="unit"): + path = Path(__file__).parent / location / "data" / filename + if not path.exists(): + return None + with open(path) as f: + return json.load(f) + + +def write_data(filename, data, location="unit"): + fpath = Path(__file__).parent / location / "data" / filename + if not fpath.exists(): + fpath.write_text(data) + + @pytest.fixture() def validate(): def schema_validate(translator, resource): @@ -15,9 +29,7 @@ def schema_validate(translator, resource): cfn = boto3.client("cloudformation") rtype = cfn.describe_type(TypeName=translator.cfn_type, Type="RESOURCE") schema = json.loads(rtype["Schema"]) - (Path(__file__).parent / "data" / schema_path).write_text( - json.dumps(schema, indent=2) - ) + write_data(schema_path, json.dumps(schema, indent=2)) props = set(resource) sprops = set(schema["properties"].keys()) @@ -40,11 +52,3 @@ def schema_validate(translator, resource): ) return schema_validate - - -def load_data(filename): - path = Path(__file__).parent / "data" / filename - if not path.exists(): - return None - with open(path) as f: - return json.load(f) diff --git a/tests/functional/terraform/aws_kinesis_stream/main.tf b/tests/functional/terraform/aws_kinesis_stream/main.tf new file mode 100644 index 0000000..4a39cae --- /dev/null +++ b/tests/functional/terraform/aws_kinesis_stream/main.tf @@ -0,0 +1,20 @@ +resource "random_pet" "name" { + length = 2 + separator = "-" +} + + +resource "aws_kinesis_stream" "test_stream" { + name = "test-${random_pet.name.id}" + shard_count = 1 + retention_period = 48 + + shard_level_metrics = [ + "IncomingBytes", + "OutgoingBytes", + ] + + tags = { + Environment = "test" + } +} diff --git a/tests/functional/test_fresources.py b/tests/functional/test_fresources.py new file mode 100644 index 0000000..552f76c --- /dev/null +++ b/tests/functional/test_fresources.py @@ -0,0 +1,22 @@ +import json + +import conftest +from pytest_terraform import terraform +from tfdevops.cli import Translator, get_state_resources + + +def get_state_path(tmpdir, tf_resources): + with open(tmpdir / "state.json", "w") as fh: + fh.write(json.dumps(tf_resources.terraform.show(), indent=2)) + return fh.name + + +@terraform("aws_kinesis_stream") +def test_kinesis_stream(tmpdir, aws_kinesis_stream, validate): + resources = get_state_resources(None, get_state_path(tmpdir, aws_kinesis_stream)) + translator = Translator.get_translator("kinesis_stream")() + props = translator.get_properties(resources["aws_kinesis_stream"][0]) + conftest.write_data( + "kinesis_stream.json", json.dumps(resources["aws_kinesis_stream"][0], indent=2) + ) + validate(translator, props) diff --git a/tests/unit/data/kinesis_stream.json b/tests/unit/data/kinesis_stream.json new file mode 100644 index 0000000..a6acde3 --- /dev/null +++ b/tests/unit/data/kinesis_stream.json @@ -0,0 +1,40 @@ +{ + "address": "aws_kinesis_stream.test_stream", + "mode": "managed", + "type": "aws_kinesis_stream", + "name": "test_stream", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 1, + "values": { + "arn": "arn:aws:kinesis:us-east-2:112233445566:stream/test-poetic-marten", + "encryption_type": "NONE", + "enforce_consumer_deletion": false, + "id": "arn:aws:kinesis:us-east-2:112233445566:stream/test-poetic-marten", + "kms_key_id": "", + "name": "test-poetic-marten", + "retention_period": 48, + "shard_count": 1, + "shard_level_metrics": [ + "IncomingBytes", + "OutgoingBytes" + ], + "tags": { + "Environment": "test" + }, + "tags_all": { + "Environment": "test" + }, + "timeouts": null + }, + "sensitive_values": { + "shard_level_metrics": [ + false, + false + ], + "tags": {}, + "tags_all": {} + }, + "depends_on": [ + "random_pet.name" + ] +} \ No newline at end of file diff --git a/tests/unit/data/schema.kinesis_stream.json b/tests/unit/data/schema.kinesis_stream.json new file mode 100644 index 0000000..9d07fd3 --- /dev/null +++ b/tests/unit/data/schema.kinesis_stream.json @@ -0,0 +1,146 @@ +{ + "typeName": "AWS::Kinesis::Stream", + "description": "Resource Type definition for AWS::Kinesis::Stream", + "sourceUrl": "https://github.com/aws-cloudformation/aws-cloudformation-resource-providers-kinesis.git", + "definitions": { + "StreamEncryption": { + "description": "When specified, enables or updates server-side encryption using an AWS KMS key for a specified stream. Removing this property from your stack template and updating your stack disables encryption.", + "type": "object", + "additionalProperties": false, + "properties": { + "EncryptionType": { + "description": "The encryption type to use. The only valid value is KMS. ", + "type": "string", + "enum": [ + "KMS" + ] + }, + "KeyId": { + "description": "The GUID for the customer-managed AWS KMS key to use for encryption. This value can be a globally unique identifier, a fully specified Amazon Resource Name (ARN) to either an alias or a key, or an alias name prefixed by \"alias/\".You can also use a master key owned by Kinesis Data Streams by specifying the alias aws/kinesis.", + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "EncryptionType", + "KeyId" + ] + }, + "Tag": { + "description": "An arbitrary set of tags (key-value pairs) to associate with the Kinesis stream.", + "type": "object", + "additionalProperties": false, + "properties": { + "Key": { + "description": "The key name of the tag. You can specify a value that is 1 to 128 Unicode characters in length and cannot be prefixed with aws:. You can use any of the following characters: the set of Unicode letters, digits, whitespace, _, ., /, =, +, and -.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "Value": { + "description": "The value for the tag. You can specify a value that is 0 to 255 Unicode characters in length and cannot be prefixed with aws:. You can use any of the following characters: the set of Unicode letters, digits, whitespace, _, ., /, =, +, and -.", + "type": "string", + "minLength": 0, + "maxLength": 255 + } + }, + "required": [ + "Key", + "Value" + ] + } + }, + "properties": { + "Arn": { + "description": "The Amazon resource name (ARN) of the Kinesis stream", + "type": "string" + }, + "Name": { + "description": "The name of the Kinesis stream.", + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-zA-Z0-9_.-]+$" + }, + "RetentionPeriodHours": { + "description": "The number of hours for the data records that are stored in shards to remain accessible.", + "type": "integer", + "minimum": 24 + }, + "ShardCount": { + "description": "The number of shards that the stream uses.", + "type": "integer", + "minimum": 1 + }, + "StreamEncryption": { + "description": "When specified, enables or updates server-side encryption using an AWS KMS key for a specified stream.", + "$ref": "#/definitions/StreamEncryption" + }, + "Tags": { + "description": "An arbitrary set of tags (key\u2013value pairs) to associate with the Kinesis stream.", + "type": "array", + "uniqueItems": false, + "insertionOrder": false, + "items": { + "$ref": "#/definitions/Tag" + } + } + }, + "additionalProperties": false, + "required": [ + "ShardCount" + ], + "readOnlyProperties": [ + "/properties/Arn" + ], + "createOnlyProperties": [ + "/properties/Name" + ], + "primaryIdentifier": [ + "/properties/Name" + ], + "handlers": { + "create": { + "permissions": [ + "kinesis:DescribeStreamSummary", + "kinesis:CreateStream", + "kinesis:IncreaseStreamRetentionPeriod", + "kinesis:StartStreamEncryption", + "kinesis:AddTagsToStream", + "kinesis:ListTagsForStream" + ] + }, + "read": { + "permissions": [ + "kinesis:DescribeStreamSummary", + "kinesis:ListTagsForStream" + ] + }, + "update": { + "permissions": [ + "kinesis:DescribeStreamSummary", + "kinesis:UpdateShardCount", + "kinesis:IncreaseStreamRetentionPeriod", + "kinesis:DecreaseStreamRetentionPeriod", + "kinesis:StartStreamEncryption", + "kinesis:StopStreamEncryption", + "kinesis:AddTagsToStream", + "kinesis:RemoveTagsFromStream", + "kinesis:ListTagsForStream" + ] + }, + "delete": { + "permissions": [ + "kinesis:DescribeStreamSummary", + "kinesis:DeleteStream", + "kinesis:RemoveTagsFromStream" + ] + }, + "list": { + "permissions": [ + "kinesis:ListStreams" + ] + } + } +} \ No newline at end of file diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index 8351c57..7bda636 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -14,3 +14,9 @@ def test_app_lb(validate): resource = load_data("app_lb.json") props = translator.get_properties(resource) validate(translator, props) + + +def test_kinesis(validate): + translator = Translator.get_translator("kinesis_stream")() + resource = load_data("kinesis_stream.json") + validate(translator, translator.get_properties(resource)) diff --git a/tfdevops/cli.py b/tfdevops/cli.py index 5709e51..0e6d1bf 100644 --- a/tfdevops/cli.py +++ b/tfdevops/cli.py @@ -597,6 +597,9 @@ def _camel_str(self, k): parts = [p.capitalize() for p in k.split("_")] return "".join(parts) + def get_tags(self, tag_map): + return [{"Key": k, "Value": v} for k, v in tag_map.items()] + def camel(self, d): r = {} @@ -765,6 +768,20 @@ def get_identity(self, r): return {self.id: r["values"]["arn"]} +class KinesisStream(Translator): + + tf_type = "kinesis_stream" + cfn_type = "AWS::Kinesis::Stream" + id = "Name" + strip = ("shard_level_metrics", "encryption_type") + rename = {"retention_period": "RetentionPeriodHours"} + + def get_properties(self, tfr): + cfr = super().get_properties(tfr) + cfr["Tags"] = self.get_tags(cfr.get("Tags", {})) + return cfr + + class Lambda(Translator): tf_type = "lambda_function" @@ -796,9 +813,7 @@ def get_properties(self, tfr): "variables" ] cfr["Code"] = {"ZipFile": tfr["values"]["filename"]} - cfr["Tags"] = [ - {"Key": k, "Value": v} for k, v in tfr["values"].get("Tags", {}).items() - ] + cfr["Tags"] = self.get_tags(tfr["values"].get("Tags", {})) if "VpcConfig" in cfr: cfr["VpcConfig"].pop("VpcId") return cfr