|
| 1 | +# -------------------------------------------------------------------------------------------- |
| 2 | +# Copyright (c) Microsoft Corporation. All rights reserved. |
| 3 | +# Licensed under the MIT License. See License.txt in the project root for license information. |
| 4 | +# -------------------------------------------------------------------------------------------- |
| 5 | +# pylint: disable=consider-using-f-string, consider-using-with, no-member |
| 6 | + |
| 7 | +import tarfile |
| 8 | +import os |
| 9 | +import re |
| 10 | +import codecs |
| 11 | +from io import open |
| 12 | +import requests |
| 13 | +from knack.log import get_logger |
| 14 | +from msrestazure.azure_exceptions import CloudError |
| 15 | +from azure.cli.core.azclierror import (CLIInternalError) |
| 16 | +from azure.cli.core.profiles import ResourceType, get_sdk |
| 17 | +from azure.cli.command_modules.acr._azure_utils import get_blob_info |
| 18 | +from azure.cli.command_modules.acr._constants import TASK_VALID_VSTS_URLS |
| 19 | + |
| 20 | +logger = get_logger(__name__) |
| 21 | + |
| 22 | + |
| 23 | +def upload_source_code(cmd, client, |
| 24 | + registry_name, |
| 25 | + resource_group_name, |
| 26 | + source_location, |
| 27 | + tar_file_path, |
| 28 | + docker_file_path, |
| 29 | + docker_file_in_tar): |
| 30 | + _pack_source_code(source_location, |
| 31 | + tar_file_path, |
| 32 | + docker_file_path, |
| 33 | + docker_file_in_tar) |
| 34 | + |
| 35 | + size = os.path.getsize(tar_file_path) |
| 36 | + unit = 'GiB' |
| 37 | + for S in ['Bytes', 'KiB', 'MiB', 'GiB']: |
| 38 | + if size < 1024: |
| 39 | + unit = S |
| 40 | + break |
| 41 | + size = size / 1024.0 |
| 42 | + |
| 43 | + logger.info("Uploading archived source code from '%s'...", tar_file_path) |
| 44 | + upload_url = None |
| 45 | + relative_path = None |
| 46 | + try: |
| 47 | + source_upload_location = client.get_build_source_upload_url( |
| 48 | + resource_group_name, registry_name) |
| 49 | + upload_url = source_upload_location.upload_url |
| 50 | + relative_path = source_upload_location.relative_path |
| 51 | + except (AttributeError, CloudError) as e: |
| 52 | + raise CLIInternalError("Failed to get a SAS URL to upload context. Error: {}".format(e.message)) from e |
| 53 | + |
| 54 | + if not upload_url: |
| 55 | + raise CLIInternalError("Failed to get a SAS URL to upload context.") |
| 56 | + |
| 57 | + account_name, endpoint_suffix, container_name, blob_name, sas_token = get_blob_info(upload_url) |
| 58 | + BlockBlobService = get_sdk(cmd.cli_ctx, ResourceType.DATA_STORAGE, 'blob#BlockBlobService') |
| 59 | + BlockBlobService(account_name=account_name, |
| 60 | + sas_token=sas_token, |
| 61 | + endpoint_suffix=endpoint_suffix, |
| 62 | + # Increase socket timeout from default of 20s for clients with slow network connection. |
| 63 | + socket_timeout=300).create_blob_from_path( |
| 64 | + container_name=container_name, |
| 65 | + blob_name=blob_name, |
| 66 | + file_path=tar_file_path) |
| 67 | + logger.info("Sending context ({0:.3f} {1}) to registry: {2}...".format( |
| 68 | + size, unit, registry_name)) |
| 69 | + return relative_path |
| 70 | + |
| 71 | + |
| 72 | +def _pack_source_code(source_location, tar_file_path, docker_file_path, docker_file_in_tar): |
| 73 | + logger.info("Packing source code into tar to upload...") |
| 74 | + |
| 75 | + original_docker_file_name = os.path.basename(docker_file_path.replace("\\", os.sep)) |
| 76 | + ignore_list, ignore_list_size = _load_dockerignore_file(source_location, original_docker_file_name) |
| 77 | + common_vcs_ignore_list = {'.git', '.gitignore', '.bzr', 'bzrignore', '.hg', '.hgignore', '.svn'} |
| 78 | + |
| 79 | + def _ignore_check(tarinfo, parent_ignored, parent_matching_rule_index): |
| 80 | + # ignore common vcs dir or file |
| 81 | + if tarinfo.name in common_vcs_ignore_list: |
| 82 | + logger.info("Excluding '%s' based on default ignore rules", tarinfo.name) |
| 83 | + return True, parent_matching_rule_index |
| 84 | + |
| 85 | + if ignore_list is None: |
| 86 | + # if .dockerignore doesn't exists, inherit from parent |
| 87 | + # eg, it will ignore the files under .git folder. |
| 88 | + return parent_ignored, parent_matching_rule_index |
| 89 | + |
| 90 | + for index, item in enumerate(ignore_list): |
| 91 | + # stop checking the remaining rules whose priorities are lower than the parent matching rule |
| 92 | + # at this point, current item should just inherit from parent |
| 93 | + if index >= parent_matching_rule_index: |
| 94 | + break |
| 95 | + if re.match(item.pattern, tarinfo.name): |
| 96 | + logger.debug(".dockerignore: rule '%s' matches '%s'.", |
| 97 | + item.rule, tarinfo.name) |
| 98 | + return item.ignore, index |
| 99 | + |
| 100 | + logger.debug(".dockerignore: no rule for '%s'. parent ignore '%s'", |
| 101 | + tarinfo.name, parent_ignored) |
| 102 | + # inherit from parent |
| 103 | + return parent_ignored, parent_matching_rule_index |
| 104 | + |
| 105 | + with tarfile.open(tar_file_path, "w:gz") as tar: |
| 106 | + # need to set arcname to empty string as the archive root path |
| 107 | + _archive_file_recursively(tar, |
| 108 | + source_location, |
| 109 | + arcname="", |
| 110 | + parent_ignored=False, |
| 111 | + parent_matching_rule_index=ignore_list_size, |
| 112 | + ignore_check=_ignore_check) |
| 113 | + |
| 114 | + # Add the Dockerfile if it's specified. |
| 115 | + # In the case of run, there will be no Dockerfile. |
| 116 | + if docker_file_path: |
| 117 | + docker_file_tarinfo = tar.gettarinfo( |
| 118 | + docker_file_path, docker_file_in_tar) |
| 119 | + with open(docker_file_path, "rb") as f: |
| 120 | + tar.addfile(docker_file_tarinfo, f) |
| 121 | + |
| 122 | + |
| 123 | +class IgnoreRule: # pylint: disable=too-few-public-methods |
| 124 | + def __init__(self, rule): |
| 125 | + |
| 126 | + self.rule = rule |
| 127 | + self.ignore = True |
| 128 | + # ! makes exceptions to exclusions |
| 129 | + if rule.startswith('!'): |
| 130 | + self.ignore = False |
| 131 | + rule = rule[1:] # remove ! |
| 132 | + # load path without leading slash in linux and windows |
| 133 | + # environments (interferes with dockerignore file) |
| 134 | + if rule.startswith('/'): |
| 135 | + rule = rule[1:] # remove beginning '/' |
| 136 | + |
| 137 | + self.pattern = "^" |
| 138 | + tokens = rule.split('/') |
| 139 | + token_length = len(tokens) |
| 140 | + for index, token in enumerate(tokens, 1): |
| 141 | + # ** matches any number of directories |
| 142 | + if token == "**": |
| 143 | + self.pattern += ".*" # treat **/ as ** |
| 144 | + else: |
| 145 | + # * matches any sequence of non-seperator characters |
| 146 | + # ? matches any single non-seperator character |
| 147 | + # . matches dot character |
| 148 | + self.pattern += token.replace( |
| 149 | + "*", "[^/]*").replace("?", "[^/]").replace(".", "\\.") |
| 150 | + if index < token_length: |
| 151 | + self.pattern += "/" # add back / if it's not the last |
| 152 | + self.pattern += "$" |
| 153 | + |
| 154 | + |
| 155 | +def _load_dockerignore_file(source_location, original_docker_file_name): |
| 156 | + # reference: https://docs.docker.com/engine/reference/builder/#dockerignore-file |
| 157 | + docker_ignore_file = os.path.join(source_location, ".dockerignore") |
| 158 | + docker_ignore_file_override = None |
| 159 | + if original_docker_file_name != "Dockerfile": |
| 160 | + docker_ignore_file_override = os.path.join( |
| 161 | + source_location, "{}.dockerignore".format(original_docker_file_name)) |
| 162 | + if os.path.exists(docker_ignore_file_override): |
| 163 | + logger.info("Overriding .dockerignore with %s", docker_ignore_file_override) |
| 164 | + docker_ignore_file = docker_ignore_file_override |
| 165 | + |
| 166 | + if not os.path.exists(docker_ignore_file): |
| 167 | + return None, 0 |
| 168 | + |
| 169 | + encoding = "utf-8" |
| 170 | + header = open(docker_ignore_file, "rb").read(len(codecs.BOM_UTF8)) |
| 171 | + if header.startswith(codecs.BOM_UTF8): |
| 172 | + encoding = "utf-8-sig" |
| 173 | + |
| 174 | + ignore_list = [] |
| 175 | + if docker_ignore_file == docker_ignore_file_override: |
| 176 | + ignore_list.append(IgnoreRule(".dockerignore")) |
| 177 | + |
| 178 | + for line in open(docker_ignore_file, 'r', encoding=encoding).readlines(): |
| 179 | + rule = line.rstrip() |
| 180 | + |
| 181 | + # skip empty line and comment |
| 182 | + if not rule or rule.startswith('#'): |
| 183 | + continue |
| 184 | + |
| 185 | + # the ignore rule at the end has higher priority |
| 186 | + ignore_list = [IgnoreRule(rule)] + ignore_list |
| 187 | + |
| 188 | + return ignore_list, len(ignore_list) |
| 189 | + |
| 190 | + |
| 191 | +def _archive_file_recursively(tar, name, arcname, parent_ignored, parent_matching_rule_index, ignore_check): |
| 192 | + # create a TarInfo object from the file |
| 193 | + tarinfo = tar.gettarinfo(name, arcname) |
| 194 | + |
| 195 | + if tarinfo is None: |
| 196 | + raise CLIInternalError("tarfile: unsupported type {}".format(name)) |
| 197 | + |
| 198 | + # check if the file/dir is ignored |
| 199 | + ignored, matching_rule_index = ignore_check( |
| 200 | + tarinfo, parent_ignored, parent_matching_rule_index) |
| 201 | + |
| 202 | + if not ignored: |
| 203 | + # append the tar header and data to the archive |
| 204 | + if tarinfo.isreg(): |
| 205 | + with open(name, "rb") as f: |
| 206 | + tar.addfile(tarinfo, f) |
| 207 | + else: |
| 208 | + tar.addfile(tarinfo) |
| 209 | + |
| 210 | + # even the dir is ignored, its child items can still be included, so continue to scan |
| 211 | + if tarinfo.isdir(): |
| 212 | + for f in os.listdir(name): |
| 213 | + _archive_file_recursively(tar, os.path.join(name, f), os.path.join(arcname, f), |
| 214 | + parent_ignored=ignored, parent_matching_rule_index=matching_rule_index, |
| 215 | + ignore_check=ignore_check) |
| 216 | + |
| 217 | + |
| 218 | +def check_remote_source_code(source_location): |
| 219 | + lower_source_location = source_location.lower() |
| 220 | + |
| 221 | + # git |
| 222 | + if lower_source_location.startswith("git@") or lower_source_location.startswith("git://"): |
| 223 | + return source_location |
| 224 | + |
| 225 | + # http |
| 226 | + if lower_source_location.startswith("https://") or lower_source_location.startswith("http://") \ |
| 227 | + or lower_source_location.startswith("github.com/"): |
| 228 | + isVSTS = any(url in lower_source_location for url in TASK_VALID_VSTS_URLS) |
| 229 | + if isVSTS or re.search(r"\.git(?:#.+)?$", lower_source_location): |
| 230 | + # git url must contain ".git" or be from VSTS/Azure DevOps. |
| 231 | + # This is because Azure DevOps doesn't follow the standard git server convention of putting |
| 232 | + # .git at the end of their URLs, so we have to special case them. |
| 233 | + return source_location |
| 234 | + if not lower_source_location.startswith("github.com/"): |
| 235 | + # Others are tarball |
| 236 | + if requests.head(source_location).status_code < 400: |
| 237 | + return source_location |
| 238 | + raise CLIInternalError("'{}' doesn't exist.".format(source_location)) |
| 239 | + |
| 240 | + # oci |
| 241 | + if lower_source_location.startswith("oci://"): |
| 242 | + return source_location |
| 243 | + raise CLIInternalError("'{}' doesn't exist.".format(source_location)) |
0 commit comments