Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 3 additions & 9 deletions deployment/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,7 @@ def run_command(command, cwd=None):
PROD_COMPOSE_FILE = "docker-compose.prod.yml"


def docker_environment(
command, base_compose_file, env_compose_file, remaining_args=None
):
def docker_environment(command, base_compose_file, env_compose_file, remaining_args=None):
"""Manage a Docker-based environment."""
if remaining_args is None:
remaining_args = []
Expand Down Expand Up @@ -125,9 +123,7 @@ def docker_environment(

def main():
"""Main entry point for the deployment script."""
parser = argparse.ArgumentParser(
description="Deployment Script for EmailIntelligence"
)
parser = argparse.ArgumentParser(description="Deployment Script for EmailIntelligence")
parser.add_argument(
"environment", choices=["dev", "staging", "prod"], help="Deployment environment"
)
Expand Down Expand Up @@ -173,9 +169,7 @@ def main():
logger.error(f"Base Docker Compose file not found: {base_file}")
sys.exit(1)
if not env_specific_file or not env_specific_file.exists():
logger.error(
f"Environment-specific Docker Compose file not found: {env_specific_file}"
)
logger.error(f"Environment-specific Docker Compose file not found: {env_specific_file}")
sys.exit(1)
else:
# This case should ideally not be reached if choices are correctly defined in argparser
Expand Down
32 changes: 8 additions & 24 deletions deployment/extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,12 @@ def get_info(self) -> Dict[str, Any]:
class ExtensionsManager:
"""Manages extensions for the EmailIntelligence application."""

def __init__(
self, root_dir: Path = ROOT_DIR, python_executable: Optional[str] = None
):
def __init__(self, root_dir: Path = ROOT_DIR, python_executable: Optional[str] = None):
"""Initialize the extensions manager."""
self.root_dir = root_dir
self.extensions_dir = root_dir / "extensions"
self.extensions: Dict[str, Extension] = {}
self.python_executable = (
python_executable if python_executable else sys.executable
)
self.python_executable = python_executable if python_executable else sys.executable
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion (code-quality): Replace if-expression with or (or-if-exp-identity)

Suggested change
self.python_executable = python_executable if python_executable else sys.executable
self.python_executable = python_executable or sys.executable


ExplanationHere we find ourselves setting a value if it evaluates to True, and otherwise
using a default.

The 'After' case is a bit easier to read and avoids the duplication of
input_currency.

It works because the left-hand side is evaluated first. If it evaluates to
true then currency will be set to this and the right-hand side will not be
evaluated. If it evaluates to false the right-hand side will be evaluated and
currency will be set to DEFAULT_CURRENCY.


def set_python_executable(self, python_executable: str):
"""Set the Python executable path."""
Expand All @@ -130,19 +126,15 @@ def discover_extensions(self) -> List[Extension]:
# Check if the extension has a metadata file
metadata_file = ext_dir / "metadata.json"
if not metadata_file.exists():
logger.warning(
f"Extension {ext_dir.name} does not have a metadata.json file"
)
logger.warning(f"Extension {ext_dir.name} does not have a metadata.json file")
continue

# Load the metadata
try:
with open(metadata_file, "r") as f:
metadata = json.load(f)
except json.JSONDecodeError as e:
logger.error(
f"Failed to parse metadata for extension {ext_dir.name}: {e}"
)
logger.error(f"Failed to parse metadata for extension {ext_dir.name}: {e}")
continue

# Check if the extension has a main module
Expand Down Expand Up @@ -198,9 +190,7 @@ def initialize_extensions(self) -> bool:
all_initialized = False
failed_extensions.append(name)
if failed_extensions:
logger.error(
f"Extensions failed to initialize: {', '.join(failed_extensions)}"
)
logger.error(f"Extensions failed to initialize: {', '.join(failed_extensions)}")
return all_initialized

def shutdown_extensions(self) -> bool:
Expand Down Expand Up @@ -283,9 +273,7 @@ def install_extension(self, url: str) -> bool:

# Clone the repository
try:
subprocess.check_call(
["git", "clone", url, str(self.extensions_dir / name)]
)
subprocess.check_call(["git", "clone", url, str(self.extensions_dir / name)])

# Check if the extension has a requirements.txt file
requirements_file = self.extensions_dir / name / "requirements.txt"
Expand Down Expand Up @@ -457,12 +445,8 @@ def shutdown():

print(f"Found {len(extensions)} extensions:")
for extension in extensions:
print(
f" {extension['name']} - {'Enabled' if extension['enabled'] else 'Disabled'}"
)
print(f" {extension['name']} - {'Enabled' if extension['enabled'] else 'Disabled'}")
print(f" Path: {extension['path']}")
print(f" Loaded: {extension['loaded']}")
print(
f" Description: {extension['metadata'].get('description', 'No description')}"
)
print(f" Description: {extension['metadata'].get('description', 'No description')}")
print()
4 changes: 1 addition & 3 deletions deployment/migrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ def rollback_migration():

def main():
"""Main entry point for the migration script."""
parser = argparse.ArgumentParser(
description="Database Migration Script for EmailIntelligence"
)
parser = argparse.ArgumentParser(description="Database Migration Script for EmailIntelligence")
parser.add_argument(
"command",
choices=["generate", "apply", "status", "rollback"],
Expand Down
8 changes: 2 additions & 6 deletions deployment/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,15 +188,11 @@ def create_placeholder_nlp_models(self) -> bool:
all_created_or_exist = True

if not placeholder_dir.exists():
logger.info(
f"Placeholder directory {placeholder_dir} does not exist. Creating it."
)
logger.info(f"Placeholder directory {placeholder_dir} does not exist. Creating it.")
try:
placeholder_dir.mkdir(parents=True, exist_ok=True)
except Exception as e:
logger.error(
f"Failed to create placeholder directory {placeholder_dir}: {e}"
)
logger.error(f"Failed to create placeholder directory {placeholder_dir}: {e}")
return False # Cannot proceed if directory cannot be created

logger.info(f"Checking for placeholder NLP models in {placeholder_dir}...")
Expand Down
8 changes: 2 additions & 6 deletions deployment/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,11 @@ def run_command(
)
if result.stdout:
logger.info(f"Command STDOUT:\n{result.stdout}")
if (
result.stderr
): # Should be empty if check=True and no error, but log if present
if result.stderr: # Should be empty if check=True and no error, but log if present
logger.warning(f"Command STDERR:\n{result.stderr}")
return True
except subprocess.CalledProcessError as e:
logger.error(
f"Command '{' '.join(e.cmd)}' failed with exit code {e.returncode}"
)
logger.error(f"Command '{' '.join(e.cmd)}' failed with exit code {e.returncode}")
if e.stdout:
logger.error(f"STDOUT:\n{e.stdout}")
if e.stderr:
Expand Down
34 changes: 7 additions & 27 deletions deployment/setup_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,7 @@ def setup_python_environment(dev_mode=False):
if dev_mode:
return run_command(f"{sys.executable} -m pip install -r requirements.txt")
else:
return run_command(
f"{sys.executable} -m pip install -r requirements.txt --no-dev"
)
return run_command(f"{sys.executable} -m pip install -r requirements.txt --no-dev")


def setup_node_environment(dev_mode=False):
Expand All @@ -82,9 +80,7 @@ def setup_database():

# Check if PostgreSQL is installed
if not run_command("psql --version"):
logger.error(
"PostgreSQL is not installed. Please install PostgreSQL and try again."
)
logger.error("PostgreSQL is not installed. Please install PostgreSQL and try again.")
return False

# Create the database if it doesn't exist
Expand Down Expand Up @@ -130,18 +126,8 @@ def setup_directories():
directories = [
PROJECT_ROOT / "deployment" / "nginx" / "ssl",
PROJECT_ROOT / "deployment" / "nginx" / "letsencrypt",
PROJECT_ROOT
/ "deployment"
/ "monitoring"
/ "grafana"
/ "provisioning"
/ "dashboards",
PROJECT_ROOT
/ "deployment"
/ "monitoring"
/ "grafana"
/ "provisioning"
/ "datasources",
PROJECT_ROOT / "deployment" / "monitoring" / "grafana" / "provisioning" / "dashboards",
PROJECT_ROOT / "deployment" / "monitoring" / "grafana" / "provisioning" / "datasources",
]

for directory in directories:
Expand All @@ -157,15 +143,9 @@ def setup_directories():

def main():
"""Main entry point for the environment setup script."""
parser = argparse.ArgumentParser(
description="Environment Setup Script for EmailIntelligence"
)
parser.add_argument(
"--dev", action="store_true", help="Set up development environment"
)
parser.add_argument(
"--force", action="store_true", help="Force overwrite of existing files"
)
parser = argparse.ArgumentParser(description="Environment Setup Script for EmailIntelligence")
parser.add_argument("--dev", action="store_true", help="Set up development environment")
parser.add_argument("--force", action="store_true", help="Force overwrite of existing files")
args = parser.parse_args()

logger.info("Setting up EmailIntelligence environment...")
Expand Down
41 changes: 10 additions & 31 deletions deployment/test_stages.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,7 @@ def run_unit_tests(self, coverage: bool = False, verbose: bool = False) -> bool:
logger.error(f"Unit tests failed with exit code {e.returncode}")
return False

def run_integration_tests(
self, coverage: bool = False, verbose: bool = False
) -> bool:
def run_integration_tests(self, coverage: bool = False, verbose: bool = False) -> bool:
"""Run integration tests."""
logger.info("Running integration tests...")

Expand Down Expand Up @@ -160,9 +158,7 @@ def run_performance_tests(
self, duration: int = 60, users: int = 10, verbose: bool = False
) -> bool:
"""Run performance tests."""
logger.info(
f"Running performance tests with {users} users for {duration} seconds..."
)
logger.info(f"Running performance tests with {users} users for {duration} seconds...")

# Ensure test dependencies are installed
if not self.env_manager.setup_environment_for_stage("test"):
Expand Down Expand Up @@ -227,9 +223,7 @@ def run_security_tests(
except subprocess.CalledProcessError:
logger.info("OWASP ZAP Python API not found, installing...")
try:
subprocess.check_call(
[python, "-m", "pip", "install", "python-owasp-zap-v2.4"]
)
subprocess.check_call([python, "-m", "pip", "install", "python-owasp-zap-v2.4"])
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

security (dangerous-subprocess-use-audit): Detected subprocess function 'check_call' without a static string. If this data can be controlled by a malicious actor, it may be an instance of command injection. Audit the use of this call to ensure it is not controllable by an external resource. You may consider using 'shlex.escape()'.

Source: opengrep

except subprocess.CalledProcessError as e:
logger.error(f"Failed to install OWASP ZAP Python API: {e}")
return False
Expand Down Expand Up @@ -286,9 +280,7 @@ def run_tests_for_stage(
elif stage == "test":
return self.run_all_tests(coverage, verbose)
elif stage == "staging":
return self.run_integration_tests(coverage, verbose) and self.run_api_tests(
verbose
)
return self.run_integration_tests(coverage, verbose) and self.run_api_tests(verbose)
elif stage == "prod":
return self.run_e2e_tests(True, verbose)
else:
Expand All @@ -306,21 +298,15 @@ def parse_arguments() -> argparse.Namespace:

# Test type arguments
parser.add_argument("--unit", action="store_true", help="Run unit tests")
parser.add_argument(
"--integration", action="store_true", help="Run integration tests"
)
parser.add_argument("--integration", action="store_true", help="Run integration tests")
parser.add_argument("--api", action="store_true", help="Run API tests")
parser.add_argument("--e2e", action="store_true", help="Run end-to-end tests")
parser.add_argument(
"--performance", action="store_true", help="Run performance tests"
)
parser.add_argument("--performance", action="store_true", help="Run performance tests")
parser.add_argument("--security", action="store_true", help="Run security tests")
parser.add_argument("--all", action="store_true", help="Run all tests")

# Test configuration
parser.add_argument(
"--coverage", action="store_true", help="Generate coverage report"
)
parser.add_argument("--coverage", action="store_true", help="Generate coverage report")
parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
parser.add_argument(
"--stage",
Expand Down Expand Up @@ -357,11 +343,7 @@ def main() -> int:

# Determine which tests to run
if args.stage:
return (
0
if test_stages.run_tests_for_stage(args.stage, args.coverage, args.verbose)
else 1
)
return 0 if test_stages.run_tests_for_stage(args.stage, args.coverage, args.verbose) else 1

if args.all:
return 0 if test_stages.run_all_tests(args.coverage, args.verbose) else 1
Expand All @@ -373,9 +355,7 @@ def main() -> int:
success = test_stages.run_unit_tests(args.coverage, args.verbose) and success

if args.integration:
success = (
test_stages.run_integration_tests(args.coverage, args.verbose) and success
)
success = test_stages.run_integration_tests(args.coverage, args.verbose) and success

if args.api:
success = test_stages.run_api_tests(args.verbose) and success
Expand All @@ -385,8 +365,7 @@ def main() -> int:

if args.performance:
success = (
test_stages.run_performance_tests(args.duration, args.users, args.verbose)
and success
test_stages.run_performance_tests(args.duration, args.users, args.verbose) and success
)

if args.security:
Expand Down
Loading