diff --git a/cmd/devenv/generate.go b/cmd/devenv/generate.go index 7d29966..23ba84c 100644 --- a/cmd/devenv/generate.go +++ b/cmd/devenv/generate.go @@ -3,12 +3,27 @@ package main import ( "fmt" "os" + "path/filepath" + "time" "github.com/spf13/cobra" "github.com/walkerlab/devenv-engine/internal/config" "github.com/walkerlab/devenv-engine/internal/templates" ) +// DeveloperJob represents work to be done for one developer +type DeveloperJob struct { + Name string +} + +// ProcessingResult represents the outcome of processing one developer +type ProcessingResult struct { + Developer string + Success bool + Error error + Duration time.Duration +} + var ( // Command-specific flags for generate outputDir string @@ -45,7 +60,7 @@ Examples: if verbose { fmt.Printf("Output directory: %s\n", outputDir) } - // TODO: implement all developers logic + generateAllDevelopersWithProgress() } else { developerName := args[0] generateSingleDeveloper(developerName) @@ -62,6 +77,129 @@ func init() { } +func generateAllDevelopersWithProgress() { + // Step 1: Discover all developers + developers, err := findAllDevelopers(configDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error discovering developers: %v\n", err) + os.Exit(1) + } + + if len(developers) == 0 { + fmt.Printf("No deveolopers found in %s\n", configDir) + return + } + + fmt.Printf("Found %d developers to process.\n", len(developers)) + + // Step 2: Set up channels for worker communication + const numWorkers = 4 + jobs := make(chan DeveloperJob, len(developers)) + results := make(chan ProcessingResult, len(developers)) + + // Step 3: Start worker goroutines + for i := 0; i < numWorkers; i++ { + go developerWorker(jobs, results) + } + + // Step 4: Send all jobs to workers + for _, dev := range developers { + jobs <- DeveloperJob{Name: dev} + } + close(jobs) + + // Step 5: Collect results + var successCount, failureCount int + var failures []ProcessingResult + + for i := 0; i < len(developers); i++ { + result := <-results + if result.Success { + successCount++ + fmt.Printf("[%d/%d] āœ… %s (%.1fs)\n", + i+1, len(developers), result.Developer, result.Duration.Seconds()) + } else { + failureCount++ + failures = append(failures, result) + fmt.Printf("[%d/%d] āŒ %s (%.1fs): %v\n", + i+1, len(developers), result.Developer, result.Duration.Seconds(), result.Error) + } + } + + // Step 6: Print final summary + fmt.Printf("\nšŸŽ‰ Batch processing complete!\n") + fmt.Printf("āœ… Successful: %d\n", successCount) + if failureCount > 0 { + fmt.Printf("āŒ Failed: %d\n", failureCount) + } + + if failureCount > 0 { + fmt.Printf("\nFailures:\n") + for _, failure := range failures { + fmt.Printf(" - %s: %v\n", failure.Developer, failure.Error) + } + os.Exit(1) // Exit with error if any failures + } +} + +func developerWorker(jobs <-chan DeveloperJob, results chan<- ProcessingResult) { + for job := range jobs { + startTime := time.Now() + err := processSingleDeveloperForBatchWithError(job.Name) + + results <- ProcessingResult{ + Developer: job.Name, + Success: err == nil, + Error: err, + Duration: time.Since(startTime), + } + } +} + +// processSingleDeveloperForBatchWithError processes a single developer for batch mode +func processSingleDeveloperForBatchWithError(developerName string) error { + if verbose { + fmt.Printf("Processing developer: %s\n", developerName) + } + + cfg, err := config.LoadDeveloperConfigWithGlobalDefaults(configDir, developerName) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Create user-specific output directory + userOutputDir := filepath.Join(outputDir, developerName) + + if !dryRun { + if err := generateManifests(cfg, userOutputDir); err != nil { + return fmt.Errorf("failed to generate manifests: %w", err) + } + } + + return nil +} + +func findAllDevelopers(configDir string) ([]string, error) { + var developers []string + + entries, err := os.ReadDir(configDir) + if err != nil { + return nil, fmt.Errorf("failed to read config directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + // Check to make sure devenv-config.yaml exists in this directory + configPath := filepath.Join(configDir, entry.Name(), "devenv-config.yaml") + if _, err := os.Stat(configPath); err == nil { + developers = append(developers, entry.Name()) + } + } + } + + return developers, nil +} + // generateSingleDeveloper handles generation for a single developer func generateSingleDeveloper(developerName string) { fmt.Printf("Generating manifests for developer: %s\n", developerName) @@ -72,6 +210,8 @@ func generateSingleDeveloper(developerName string) { fmt.Printf("Dry run mode: %t\n", dryRun) } + userOutputDir := filepath.Join(outputDir, developerName) + cfg, err := config.LoadDeveloperConfigWithGlobalDefaults(configDir, developerName) if err != nil { fmt.Fprintf(os.Stderr, "Error loading config for developer %s: %v\n", developerName, err) @@ -85,12 +225,12 @@ func generateSingleDeveloper(developerName string) { } if !dryRun { - if err := generateManifests(cfg, outputDir); err != nil { + if err := generateManifests(cfg, userOutputDir); err != nil { fmt.Fprintf(os.Stderr, "Error generating manifests: %v\n", err) os.Exit(1) } } else { - fmt.Printf("šŸ” Dry run - would generate manifests to: %s\n", outputDir) + fmt.Printf("šŸ” Dry run - would generate manifests to: %s\n", userOutputDir) } } diff --git a/internal/config/types.go b/internal/config/types.go index e108bb7..227897d 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -27,7 +27,7 @@ type BaseConfig struct { InstallHomebrew bool `yaml:"installHomebrew,omitempty"` ClearLocalPackages bool `yaml:"clearLocalPackages,omitempty"` ClearVSCodeCache bool `yaml:"clearVSCodeCache,omitempty"` - PythonBinPath string `yaml:"pythonBinPath,omitempty" validate:"omitempty,min=1"` + PythonBinPath string `yaml:"pythonBinPath,omitempty" validate:"omitempty,min=1,filepath"` } // DevEnvConfig represents the complete configuration for a developer environment. diff --git a/internal/templates/files/env-setup.tmpl b/internal/templates/files/env-setup.tmpl deleted file mode 100644 index 3204a86..0000000 --- a/internal/templates/files/env-setup.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: env-setup-{{.Name}} - labels: - app: devenv-{{.Name}} -data: - # Templated user setup script - processed with config values - setup.sh: | -{{getTemplatedScript "user-setup.sh" . | indent 4}} \ No newline at end of file diff --git a/internal/templates/files/startup-scripts.tmpl b/internal/templates/files/startup-scripts.tmpl index 29c9f1c..f5f47f1 100644 --- a/internal/templates/files/startup-scripts.tmpl +++ b/internal/templates/files/startup-scripts.tmpl @@ -15,4 +15,8 @@ data: # Static requirements file requirements.txt: | -{{getStaticScript "requirements.txt" | indent 4}} \ No newline at end of file +{{getStaticScript "requirements.txt" | indent 4}} + + # User setup script + setup.sh: | +{{getTemplatedScript "user-setup.sh" . | indent 4}} \ No newline at end of file diff --git a/internal/templates/files/statefulset.tmpl b/internal/templates/files/statefulset.tmpl index 0146154..3e86f65 100644 --- a/internal/templates/files/statefulset.tmpl +++ b/internal/templates/files/statefulset.tmpl @@ -95,9 +95,6 @@ spec: - name: startup-scripts mountPath: /scripts readOnly: true - - name: env-setup - mountPath: /env-setup - readOnly: true - name: ssh-keys mountPath: /home/{{.Name}}/.ssh readOnly: true @@ -119,10 +116,6 @@ spec: configMap: name: startup-scripts-{{.Name}} defaultMode: 0755 - - name: env-setup - configMap: - name: env-setup-{{.Name}} - defaultMode: 0755 - name: ssh-keys secret: secretName: ssh-keys-{{.Name}} diff --git a/internal/templates/renderer.go b/internal/templates/renderer.go index 5fa5bda..f3a88f5 100644 --- a/internal/templates/renderer.go +++ b/internal/templates/renderer.go @@ -113,7 +113,7 @@ func (r *Renderer) RenderTemplate(templateName string, config *config.DevEnvConf } func (r *Renderer) RenderAll(config *config.DevEnvConfig) error { - templatesToRender := []string{"statefulset", "service", "env-vars", "secret", "startup-scripts", "env-setup"} + templatesToRender := []string{"statefulset", "service", "env-vars", "secret", "startup-scripts"} for _, templateName := range templatesToRender { if err := r.RenderTemplate(templateName, config); err != nil { return fmt.Errorf("failed to render template %s: %w", templateName, err) diff --git a/internal/templates/scripts/templated/startup.sh b/internal/templates/scripts/templated/startup.sh index eb1e570..d72e25c 100644 --- a/internal/templates/scripts/templated/startup.sh +++ b/internal/templates/scripts/templated/startup.sh @@ -149,7 +149,7 @@ echo "Section 6: Package installation complete" # === USER ENVIRONMENT SETUP === # Set up environment for the user -if [ -f /env-setup/setup.sh ]; then +if [ -f /scripts/setup.sh ]; then echo "Running user environment setup script" sudo -u ${DEV_USERNAME} \ GIT_USER_NAME="{{.Git.Name}}" \ @@ -157,7 +157,7 @@ if [ -f /env-setup/setup.sh ]; then ENV_BASH_SCRIPT=${ENV_BASH_SCRIPT} \ ENV_INIT_SCRIPT=${ENV_INIT_SCRIPT} \ PYTHON_BIN_PATH=${PYTHON_BIN_PATH} \ - bash /env-setup/setup.sh + bash /scripts/setup.sh fi if [ -f "${ENV_INIT_SCRIPT}" ]; then @@ -175,9 +175,7 @@ echo "Clearing VSCode server cache" rm -rf /home/${DEV_USERNAME}/.vscode-server/ {{- end}} -# Add default VSCode remote machine config, setting default python path -mkdir -p /home/${DEV_USERNAME}/.vscode-server/data/Machine -echo "{\"python.defaultInterpreterPath\": \"${PYTHON_PATH}\"}" > /home/${DEV_USERNAME}/.vscode-server/data/Machine/settings.json + # Make sure .vscode-server directory is owned by ${DEV_USERNAME} chown -R ${DEV_USERNAME}:${DEV_USERNAME} /home/${DEV_USERNAME}/.vscode-server diff --git a/internal/templates/testdata/golden/startup-scripts.yaml b/internal/templates/testdata/golden/startup-scripts.yaml index d04fa12..fd8370f 100644 --- a/internal/templates/testdata/golden/startup-scripts.yaml +++ b/internal/templates/testdata/golden/startup-scripts.yaml @@ -123,7 +123,7 @@ data: # === USER ENVIRONMENT SETUP === # Set up environment for the user - if [ -f /env-setup/setup.sh ]; then + if [ -f /scripts/setup.sh ]; then echo "Running user environment setup script" sudo -u ${DEV_USERNAME} \ GIT_USER_NAME="Test User" \ @@ -131,7 +131,7 @@ data: ENV_BASH_SCRIPT=${ENV_BASH_SCRIPT} \ ENV_INIT_SCRIPT=${ENV_INIT_SCRIPT} \ PYTHON_BIN_PATH=${PYTHON_BIN_PATH} \ - bash /env-setup/setup.sh + bash /scripts/setup.sh fi if [ -f "${ENV_INIT_SCRIPT}" ]; then @@ -145,9 +145,7 @@ data: # === VSCODE CONFIGURATION === - # Add default VSCode remote machine config, setting default python path - mkdir -p /home/${DEV_USERNAME}/.vscode-server/data/Machine - echo "{\"python.defaultInterpreterPath\": \"${PYTHON_PATH}\"}" > /home/${DEV_USERNAME}/.vscode-server/data/Machine/settings.json + # Make sure .vscode-server directory is owned by ${DEV_USERNAME} chown -R ${DEV_USERNAME}:${DEV_USERNAME} /home/${DEV_USERNAME}/.vscode-server @@ -244,4 +242,86 @@ data: openpyxl>=3.1.0 # Machine learning (common packages) - scikit-learn>=1.3.0 \ No newline at end of file + scikit-learn>=1.3.0 + + # User setup script + setup.sh: | +#!/bin/bash + # User environment setup script for: testuser + # This script runs as the developer user to configure their personal environment + set -e + + # Get script file names from environment + INIT_SCRIPT_NAME=$(basename "${ENV_INIT_SCRIPT}") + BASH_SCRIPT_NAME=$(basename "${ENV_BASH_SCRIPT}") + + echo "Setting up user environment for testuser" + + # === BASHRC SETUP === + cat > ~/.bashrc << 'EOF_BASHRC' + + # Add the Python bin path to the PATH + # Ensure this takes precedence over Homebrew + export PATH=":${PATH}" + + # Custom aliases + alias ll='ls -la' + + # GPU-related settings + export CUDA_DEVICE_ORDER=PCI_BUS_ID + + # Run env bash script if it exists + if [ -f "${ENV_BASH_SCRIPT}" ]; then + echo "Running ${ENV_BASH_SCRIPT}" >&2 + source ${ENV_BASH_SCRIPT} + fi + + # Welcome message + echo " " >&2 + echo "ā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā•— ā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā•— ā–ˆā–ˆā•— ā–ˆā–ˆā–ˆā•— ā–ˆā–ˆā•— ā–ˆā–ˆā•—" >&2 + echo "ā–ˆā–ˆā•”ā•ā•ā–ˆā–ˆā•— ā•šā•ā•ā•ā•ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ ā–ˆā–ˆā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘" >&2 + echo "ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ ā–ˆā–ˆā–ˆā–ˆā•— ā–ˆā–ˆā•— ā–ˆā–ˆā•— ā–ˆā–ˆā–ˆā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ ā–ˆā–ˆ ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘" >&2 + echo "ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ ā–ˆā–ˆā–„ā–„ā–„ā–ˆā•—ā•šā–ˆā–ˆā•— ā–ˆā–ˆā•”ā• ā•šā•ā•ā–ˆā–ˆā•‘ ā–ˆā–ˆā•‘ā–ˆā–ˆā•”ā•ā–ˆā–ˆā•‘ ā•šā–ˆā–ˆā•— ā–ˆā–ˆā•”ā•" >&2 + echo "ā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā•”ā• ā–ˆā–ˆā–„ā–„ā–„ā–„ā•— ā•šā–ˆā–ˆā–ˆā–ˆā•”ā• ā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā–ˆā•‘ ā–ˆā–ˆā–ˆā–ˆā•”ā• ā–ˆā–ˆā•‘ ā•šā–ˆā–ˆā–ˆā–ˆā•”ā• " >&2 + echo "ā•šā•ā•ā•ā•ā•ā• ā•šā•ā•ā•ā•ā•ā• ā•šā•ā•ā•ā• ā•šā•ā•ā•ā•ā•ā•ā• ā•šā•ā•ā•ā• ā•šā•ā• ā•šā•ā•ā•ā• " >&2 + echo " " >&2 + echo "Welcome to your ENIGMA DevENV, testuser!" >&2 + echo "Place ${INIT_SCRIPT_NAME} in your home directory to customize DevENV initialization." >&2 + echo "Edit ${BASH_SCRIPT_NAME} to customize your Shell environment" >&2 + echo "" >&2 + echo "If you encounter issues, please contact the administrator." >&2 + echo "Happy dev'ing!" >&2 + + EOF_BASHRC + + # === BASH PROFILE SETUP === + echo "source ~/.bashrc" > ~/.bash_profile + + # === CREATE CUSTOM BASH SCRIPT === + # Create env bash script file if it doesn't exist + if [ ! -f "${ENV_BASH_SCRIPT}" ]; then + cat > "${ENV_BASH_SCRIPT}" << 'EOF_BASH_CUSTOM' + #!/bin/bash + # Custom bash environment configuration + + # Add your custom environment variables and aliases here + EOF_BASH_CUSTOM + chmod +x "${ENV_BASH_SCRIPT}" + fi + + # === JUPYTER CONFIGURATION === + mkdir -p ~/.jupyter + if [ ! -f ~/.jupyter/jupyter_notebook_config.py ]; then + cat > ~/.jupyter/jupyter_notebook_config.py << 'EOF_JUPYTER' + c.NotebookApp.ip = '0.0.0.0' + c.NotebookApp.open_browser = False + c.NotebookApp.port = 8888 + EOF_JUPYTER + fi + + # === GIT CONFIGURATION === + echo "Configuring Git for testuser" + git config --global user.name "${GIT_USER_NAME}" + git config --global user.email "${GIT_USER_EMAIL}" + + echo "User environment setup complete for testuser" \ No newline at end of file diff --git a/internal/templates/testdata/golden/statefulset.yaml b/internal/templates/testdata/golden/statefulset.yaml index 9d7843d..0fa79f3 100644 --- a/internal/templates/testdata/golden/statefulset.yaml +++ b/internal/templates/testdata/golden/statefulset.yaml @@ -72,9 +72,6 @@ spec: - name: startup-scripts mountPath: /scripts readOnly: true - - name: env-setup - mountPath: /env-setup - readOnly: true - name: ssh-keys mountPath: /home/testuser/.ssh readOnly: true @@ -96,10 +93,6 @@ spec: configMap: name: startup-scripts-testuser defaultMode: 0755 - - name: env-setup - configMap: - name: env-setup-testuser - defaultMode: 0755 - name: ssh-keys secret: secretName: ssh-keys-testuser