diff --git a/README.md b/README.md index 75f8188..ed72f66 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ # Snagsby [![Build Status](https://travis-ci.org/roverdotcom/snagsby.svg?branch=master)](https://travis-ci.org/roverdotcom/snagsby) -Snagsby reads a JSON object from an S3 bucket and outputs the keys and values -in a format that can be evaluated by a shell to set environment variables. +Snagsby reads configuration and secrets from multiple sources and outputs them +as environment variables in a format that can be evaluated by a shell. -It's useful for reading configuration into environment variables from S3 -objects in Docker containers. +**Supported sources:** +- Local env files (`file://`) with dotenv format +- AWS S3 JSON objects (`s3://`) +- AWS Secrets Manager (`sm://`) + +It's useful for reading configuration and secrets into environment variables in +Docker containers and other deployment scenarios. It can help with workflows like this one: https://blogs.aws.amazon.com/security/post/Tx2B3QUWAA7KOU/How-to-Manage-Secrets-for-Amazon-EC2-Container-Service-Based-Applications-by-Usi @@ -55,10 +60,11 @@ export YES="1" You can supply sources in a comma delimited `SNAGSBY_SOURCE` environment variable: ```bash -SNAGSBY_SOURCE="s3://my-bucket/secrets1.json, s3://my-bucket/secrets2.json" ./bin/snagsby +SNAGSBY_SOURCE="file://base.snagsby, s3://my-bucket/secrets.json" ./bin/snagsby # -e will fail on errors and exit 1 ./bin/snagsby -e \ + file://production.snagsby \ s3://my-bucket/config.json \ s3://my-bucket/config2.json ``` @@ -70,13 +76,103 @@ An example docker entrypoint may look like: set -e +# Combine local config with remote secrets eval $(./bin/snagsby \ - s3://my-bucket/config.json?region=us-west-2 \ - s3://my-bucket/config-production.json?region-us-west-1) + file://config/base.snagsby \ + file://config/production.snagsby \ + s3://my-bucket/config.json?region=us-west-2) exec "$@" ``` +## Env File Format + +Snagsby supports reading environment variables from local files using the `file://` scheme with standard dotenv format. + +### Basic Usage + +```bash +snagsby file://local.snagsby +``` + +### File Format + +Env files use the standard dotenv format (`KEY=VALUE`): + +```bash +# Comments are supported +DATABASE_URL=postgres://localhost:5432/mydb +API_KEY=abc123 +DEBUG=true + +# Values can be quoted to preserve special characters +MESSAGE="Hello # this is not a comment" +PATH_WITH_SPACES=" /path/with/spaces " + +# Empty values are allowed +OPTIONAL_KEY= +``` + +### Secret References + +Values can reference AWS Secrets Manager using the `sm://` prefix: + +```bash +# Direct values +DATABASE_HOST=localhost +DATABASE_PORT=5432 + +# References to secrets in AWS Secrets Manager +DATABASE_PASSWORD=sm://production/db/password +API_SECRET=sm://production/api/secret +``` + +Snagsby will automatically fetch the secrets from AWS Secrets Manager and populate the environment variables with the actual values. + +### File Naming Conventions + +While Snagsby accepts any file extension, we recommend using extensions that clearly indicate the file contains **secret references**, not actual secrets: + +- `.snagsby` - Clear about the tool used to resolve the secrets +- `.env.vault` - Suggests secrets/vault references +- `.env.ref` - Short for "references" +- `.envmap` - Conveys "mapping to secrets" + +It is recommended to **avoid using `.env`** for files with secret references, as it may give developers a false sense that the file is safe to commit with actual secrets or that the file will not be committed to the repository. + +### Multiple Sources + +You can combine multiple source types: + +```bash +snagsby \ + file://base.snagsby \ + file://production.snagsby \ + s3://my-bucket/config.json?region=us-west-2 +``` + +### Validation and Key Handling + +**Env files require strict POSIX-compliant variable names:** + +Environment variable names in env files must follow shell naming conventions: +- Start with a letter or underscore (`[a-zA-Z_]`) +- Contain only letters, digits, and underscores (`[a-zA-Z0-9_]`) + +Invalid keys (e.g., `my-key` with dashes, `my.key` with dots, or `123key` starting with a digit) will be rejected with a clear error message. + +**Why strict validation?** + +Unlike other Snagsby resolvers (S3, Secrets Manager) that normalize arbitrary keys (e.g., converting `my-key` to `MY_KEY`), env files are meant to define actual shell environment variables. Since shells only accept POSIX-compliant names, we validate at parse time to catch errors early. + +**Key preservation:** + +Keys in env files are preserved exactly as written, including their case. For example: +- `DATABASE_URL=...` stays as `DATABASE_URL` (not normalized to uppercase) +- `lowercase_var=...` stays as `lowercase_var` + +This matches standard `.env` file behavior and ensures variables are set exactly as intended. + ## AWS Configuration You can configure AWS any way the golang sdk supports: diff --git a/pkg/connectors/secretsmanager.go b/pkg/connectors/secretsmanager.go index e5a2189..ff68d5b 100644 --- a/pkg/connectors/secretsmanager.go +++ b/pkg/connectors/secretsmanager.go @@ -27,6 +27,9 @@ type SecretsManagerAPIClient interface { GetSecretValueAPIClient } +// SecretsManagerConnector provides methods for retrieving secrets from AWS Secrets Manager. +// The struct fields are private to prevent direct instantiation outside this package. +// Use NewSecretsManagerConnector to create instances. type SecretsManagerConnector struct { secretsmanagerClient SecretsManagerAPIClient source *config.Source @@ -40,6 +43,12 @@ func NewSecretsManagerConnector(source *config.Source) (*SecretsManagerConnector return &SecretsManagerConnector{secretsmanagerClient: secretsManagerClient, source: source}, nil } +// NewSecretsManagerConnectorWithClient creates a new SecretsManagerConnector with a custom API client. +// This is primarily used for testing to inject a mock client. Production code should use NewSecretsManagerConnector instead. +func NewSecretsManagerConnectorWithClient(client SecretsManagerAPIClient, source *config.Source) *SecretsManagerConnector { + return &SecretsManagerConnector{secretsmanagerClient: client, source: source} +} + func (sm *SecretsManagerConnector) getConcurrencyOrDefault(keyLength int) int { // Pull concurrency settings getConcurrency, hasSetting := os.LookupEnv("SNAGSBY_SM_CONCURRENCY") @@ -72,7 +81,7 @@ func (sm *SecretsManagerConnector) fetchSecretValue(secretName string) (string, getSecret, err := sm.secretsmanagerClient.GetSecretValue(ctx, input) if err != nil { - return "", err + return "", fmt.Errorf("fetching secret %q: %w", secretName, err) } if getSecret.SecretString == nil { diff --git a/pkg/connectors/secretsmanager_test.go b/pkg/connectors/secretsmanager_test.go index 869b3c0..1e7924e 100644 --- a/pkg/connectors/secretsmanager_test.go +++ b/pkg/connectors/secretsmanager_test.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "strconv" + "strings" "sync/atomic" "testing" "time" @@ -160,19 +161,19 @@ func TestGetSecretWithVersionID(t *testing.T) { // TestGetSecretErrors tests error handling in GetSecret func TestGetSecretErrors(t *testing.T) { tests := []struct { - name string - mockBehavior mockAWSSecretsManagerBehavior - expectedError string + name string + mockBehavior mockAWSSecretsManagerBehavior + expectedErrorSubstring string }{ { - name: "access denied error", - mockBehavior: errorBehavior(errors.New("access denied")), - expectedError: "access denied", + name: "access denied error", + mockBehavior: errorBehavior(errors.New("access denied")), + expectedErrorSubstring: "access denied", }, { - name: "secret not found error", - mockBehavior: errorBehavior(errors.New("secret not found")), - expectedError: "secret not found", + name: "secret not found error", + mockBehavior: errorBehavior(errors.New("secret not found")), + expectedErrorSubstring: "secret not found", }, } @@ -187,8 +188,8 @@ func TestGetSecretErrors(t *testing.T) { if err == nil { t.Error("Expected error but got none") - } else if err.Error() != tt.expectedError { - t.Errorf("Expected error '%s', got '%s'", tt.expectedError, err.Error()) + } else if !strings.Contains(err.Error(), tt.expectedErrorSubstring) { + t.Errorf("Expected error containing '%s', got '%s'", tt.expectedErrorSubstring, err.Error()) } }) } diff --git a/pkg/connectors/testing/mocks.go b/pkg/connectors/testing/mocks.go index 55d8bf2..b4c23bc 100644 --- a/pkg/connectors/testing/mocks.go +++ b/pkg/connectors/testing/mocks.go @@ -1,5 +1,16 @@ package testing +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/roverdotcom/snagsby/pkg/config" + "github.com/roverdotcom/snagsby/pkg/connectors" +) + // MockSecretsConnector is a reusable mock for any connector that implements secret retrieval methods. // It can be used by all resolvers (manifest, envfile, secretsmanager, etc.) that need to mock secret operations. // @@ -45,3 +56,74 @@ func (m *MockSecretsConnector) ListSecrets(prefix string) ([]string, error) { } return []string{}, nil } + +// MockSecretsManagerAPIClient is a mock implementation of the AWS Secrets Manager API client. +// This allows testing the full integration of Resolver + Connector with a mocked AWS SDK client. +// +// Usage example: +// +// mock := &testing.MockSecretsManagerAPIClient{ +// Secrets: map[string]string{ +// "path/to/secret": "secret-value", +// }, +// } +type MockSecretsManagerAPIClient struct { + // Secrets maps secret names to their string values + Secrets map[string]string + + // GetSecretValueFunc allows custom behavior for GetSecretValue + GetSecretValueFunc func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) + + // ListSecretsFunc allows custom behavior for ListSecrets + ListSecretsFunc func(ctx context.Context, params *secretsmanager.ListSecretsInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.ListSecretsOutput, error) +} + +// GetSecretValue implements the GetSecretValueAPIClient interface. +func (m *MockSecretsManagerAPIClient) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + if m.GetSecretValueFunc != nil { + return m.GetSecretValueFunc(ctx, params, optFns...) + } + + if m.Secrets == nil { + return nil, fmt.Errorf("ResourceNotFoundException: secret not found") + } + + secretName := aws.ToString(params.SecretId) + value, exists := m.Secrets[secretName] + if !exists { + return nil, fmt.Errorf("ResourceNotFoundException: secret %s not found", secretName) + } + + return &secretsmanager.GetSecretValueOutput{ + SecretString: aws.String(value), + }, nil +} + +// ListSecrets implements the ListSecretsAPIClient interface. +func (m *MockSecretsManagerAPIClient) ListSecrets(ctx context.Context, params *secretsmanager.ListSecretsInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.ListSecretsOutput, error) { + if m.ListSecretsFunc != nil { + return m.ListSecretsFunc(ctx, params, optFns...) + } + + return &secretsmanager.ListSecretsOutput{ + SecretList: []types.SecretListEntry{}, + }, nil +} + +// NewSecretsManagerConnectorWithFakeSecrets creates a SecretsManagerConnector with fake secrets for testing. +// This is a convenience function that encapsulates the mock setup so tests don't need to know about the mock client implementation. +// +// Usage example: +// +// connector := testing.NewSecretsManagerConnectorWithFakeSecrets( +// map[string]string{ +// "path/to/secret": "secret-value", +// }, +// source, +// ) +func NewSecretsManagerConnectorWithFakeSecrets(secrets map[string]string, source *config.Source) *connectors.SecretsManagerConnector { + mockClient := &MockSecretsManagerAPIClient{ + Secrets: secrets, + } + return connectors.NewSecretsManagerConnectorWithClient(mockClient, source) +} diff --git a/pkg/resolvers/envfile.go b/pkg/resolvers/envfile.go new file mode 100644 index 0000000..0645f04 --- /dev/null +++ b/pkg/resolvers/envfile.go @@ -0,0 +1,212 @@ +package resolvers + +import ( + "bufio" + "fmt" + "io" + "maps" + "os" + "regexp" + "slices" + "strings" + + "github.com/roverdotcom/snagsby/pkg/config" +) + +// envVarNameRegexp validates POSIX-compliant environment variable names. +// Must start with letter or underscore, followed by letters, digits, or underscores. +var envVarNameRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +type envFileSecretsGetter interface { + GetSecrets(keys []string) (map[string]string, []error) +} + +type EnvFileResolver struct { + connector envFileSecretsGetter +} + +func NewEnvFileResolver(connector envFileSecretsGetter) *EnvFileResolver { + return &EnvFileResolver{connector: connector} +} + +// isValidEnvVarName checks if a key is a valid POSIX environment variable name. +// Rejects keys with dashes, dots, or starting with digits. +func isValidEnvVarName(key string) bool { + return envVarNameRegexp.MatchString(key) +} + +func getFilePath(source *config.Source) string { + // Absolute path: file:///absolute/path -> Host="", Path="/absolute/path" + if source.URL.Host == "" { + return source.URL.Path + } + + // Relative path with only filename: file://filename -> Host="filename", Path="" + if source.URL.Path == "" { + return source.URL.Host + } + + // Relative path: file://./path or file://../path -> Host="." or "..", Path="/path" + // Simply concatenate host and path (path already has leading slash) + return source.URL.Host + source.URL.Path +} + +// parseEnvLine parses a single line from an env file into a key-value pair. +// Keys must be valid POSIX environment variable names (start with letter/underscore, +// contain only alphanumerics and underscores). Keys are preserved exactly as written. +// Quoted values (using " or ') preserve their content including hashes and whitespace. +// Returns empty strings for blank lines or comment-only lines (no error). +func parseEnvLine(line string) (string, string, error) { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" || strings.HasPrefix(trimmedLine, "#") { + return "", "", nil + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid line: %s", line) + } + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + if key == "" { + return "", "", fmt.Errorf("invalid line: %s (empty key)", line) + } + + if !isValidEnvVarName(key) { + return "", "", fmt.Errorf("invalid key '%s': environment variable names must contain only letters, digits, and underscores, and must start with a letter or underscore", key) + } + + // Handle quoted values (both single and double quotes) + if len(value) >= 1 { + firstChar := value[0] + if firstChar == '"' || firstChar == '\'' { + // Find the closing quote + closingQuoteIdx := strings.IndexByte(value[1:], firstChar) + if closingQuoteIdx == -1 { + // No closing quote found + return key, "", fmt.Errorf("invalid line: %s (uneven quotes)", line) + } + // closingQuoteIdx is relative to value[1:], so actual index is closingQuoteIdx + 1 + actualClosingIdx := closingQuoteIdx + 1 + // Extract the value between quotes + quotedValue := value[1:actualClosingIdx] + // Everything after the closing quote is ignored (including comments) + return key, quotedValue, nil + } + } + + // Remove inline comments for unquoted values + if idx := strings.Index(value, " #"); idx != -1 { + value = strings.TrimSpace(value[:idx]) + } + return key, value, nil +} + +// parsedEnvFile holds the parsed environment variables from a file. +type parsedEnvFile struct { + envVars map[string]string + envVarsOrder []string + needsResolution map[string]string +} + +// parseEnvFile reads and parses an env file, identifying variables and secrets. +// Returns parsed data structure with env vars, their order, and secrets needing resolution. +func parseEnvFile(file io.Reader, result *Result) *parsedEnvFile { + parsed := &parsedEnvFile{ + envVars: make(map[string]string), + envVarsOrder: []string{}, + needsResolution: map[string]string{}, + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + key, value, err := parseEnvLine(line) + if err != nil { + result.AppendError(err) + continue + } + + if key == "" { + continue + } + + // Check for duplicate keys + if _, exists := parsed.envVars[key]; exists { + result.AppendError(fmt.Errorf("duplicate key '%s' found in env file, duplicate keys are not supported", key)) + continue + } + + parsed.envVars[key] = value + parsed.envVarsOrder = append(parsed.envVarsOrder, key) + + // If the value points to sm, we will need to resolve it before we can add it to the result + if strings.HasPrefix(value, "sm://") { + parsed.needsResolution[key] = strings.TrimPrefix(value, "sm://") + } + } + if err := scanner.Err(); err != nil { + result.AppendError(err) + } + + return parsed +} + +// populateResultWithSecrets adds environment variables to the result, resolving secrets as needed. +func populateResultWithSecrets(parsed *parsedEnvFile, secrets map[string]string, result *Result) { + for _, key := range parsed.envVarsOrder { + secretPath, needsSecret := parsed.needsResolution[key] + if needsSecret { + if secretValue, found := secrets[secretPath]; found { + result.AppendItemExact(key, secretValue) + } + // If secret not found, skip it (error already reported during GetSecrets) + } else { + result.AppendItemExact(key, parsed.envVars[key]) + } + } +} + +func (e *EnvFileResolver) resolve(file io.Reader, result *Result) { + parsed := parseEnvFile(file, result) + + // All lines have explicit values. No need to resolve them. + if len(parsed.needsResolution) == 0 { + for _, key := range parsed.envVarsOrder { + result.AppendItemExact(key, parsed.envVars[key]) + } + return + } + + // The values in the original env file contain the path for secrets manager + // Dedupe secret keys to avoid redundant API calls + secretKeysMap := make(map[string]bool) + for _, secretPath := range parsed.needsResolution { + secretKeysMap[secretPath] = true + } + secretKeys := slices.Collect(maps.Keys(secretKeysMap)) + secrets, errors := e.connector.GetSecrets(secretKeys) + + for _, err := range errors { + result.AppendError(err) + } + + populateResultWithSecrets(parsed, secrets, result) +} + +func (e *EnvFileResolver) Resolve(source *config.Source) *Result { + result := &Result{Source: source} + + filePath := getFilePath(source) + fileReader, err := os.Open(filePath) + if err != nil { + result.AppendError(err) + return result + } + defer fileReader.Close() + + e.resolve(fileReader, result) + + return result +} diff --git a/pkg/resolvers/envfile_test.go b/pkg/resolvers/envfile_test.go new file mode 100644 index 0000000..c974da2 --- /dev/null +++ b/pkg/resolvers/envfile_test.go @@ -0,0 +1,647 @@ +package resolvers + +import ( + "fmt" + "net/url" + "os" + "strings" + "testing" + + "github.com/roverdotcom/snagsby/pkg/config" + connectortesting "github.com/roverdotcom/snagsby/pkg/connectors/testing" +) + +func TestGetFilePath(t *testing.T) { + examples := []struct { + name string + urlString string + expectedPath string + }{ + { + name: "absolute path with file scheme", + urlString: "file:///absolute/path/to/file.env", + expectedPath: "/absolute/path/to/file.env", + }, + { + name: "relative path with current directory", + urlString: "file://./pre-cache.env", + expectedPath: "./pre-cache.env", + }, + { + name: "relative path with parent directory", + urlString: "file://../parent/file.env", + expectedPath: "../parent/file.env", + }, + { + name: "relative path with 2 levels up directory", + urlString: "file://../../parent/file.env", + expectedPath: "../../parent/file.env", + }, + { + name: "simple filename", + urlString: "file://local.env", + expectedPath: "local.env", + }, + } + + for _, example := range examples { + t.Run(example.name, func(t *testing.T) { + parsedURL, err := url.Parse(example.urlString) + if err != nil { + t.Fatalf("Failed to parse URL: %v", err) + } + source := &config.Source{URL: parsedURL} + actualPath := getFilePath(source) + if actualPath != example.expectedPath { + t.Errorf("Expected path '%s' but got '%s' (URL.Host='%s', URL.Path='%s')", + example.expectedPath, actualPath, parsedURL.Host, parsedURL.Path) + } + }) + } +} + +func TestProcessLine(t *testing.T) { + examples := []struct { + name string + line string + expectedKey string + expectedValue string + expectedError error + }{ + { + name: "empty line", + line: "", + expectedKey: "", + expectedValue: "", + expectedError: nil, + }, + { + name: "comment line", + line: "# This is a comment", + expectedKey: "", + expectedValue: "", + expectedError: nil, + }, + { + name: "simple key value", + line: "FOO=bar", + expectedKey: "FOO", + expectedValue: "bar", + expectedError: nil, + }, + { + name: "key value with spaces", + line: " FOO = bar ", + expectedKey: "FOO", + expectedValue: "bar", + expectedError: nil, + }, + { + name: "key value with inline comment", + line: "FOO=bar # This is a comment", + expectedKey: "FOO", + expectedValue: "bar", + expectedError: nil, + }, + { + name: "invalid line without equals", + line: "FOO bar", + expectedKey: "", + expectedValue: "", + expectedError: fmt.Errorf("invalid line: FOO bar"), + }, + { + name: "empty key with value", + line: "=value", + expectedKey: "", + expectedValue: "", + expectedError: fmt.Errorf("invalid line: =value (empty key)"), + }, + { + name: "whitespace key with value", + line: " =value", + expectedKey: "", + expectedValue: "", + expectedError: fmt.Errorf("invalid line: =value (empty key)"), + }, + { + name: "key with dash should fail", + line: "foo-bar=value", + expectedKey: "", + expectedValue: "", + expectedError: fmt.Errorf("invalid key 'foo-bar': environment variable names must contain only letters, digits, and underscores, and must start with a letter or underscore"), + }, + { + name: "key with dot should fail", + line: "foo.bar=value", + expectedKey: "", + expectedValue: "", + expectedError: fmt.Errorf("invalid key 'foo.bar': environment variable names must contain only letters, digits, and underscores, and must start with a letter or underscore"), + }, + { + name: "key starting with digit should fail", + line: "123FOO=value", + expectedKey: "", + expectedValue: "", + expectedError: fmt.Errorf("invalid key '123FOO': environment variable names must contain only letters, digits, and underscores, and must start with a letter or underscore"), + }, + { + name: "valid key with underscore", + line: "FOO_BAR=value", + expectedKey: "FOO_BAR", + expectedValue: "value", + expectedError: nil, + }, + { + name: "valid key lowercase", + line: "foo_bar=value", + expectedKey: "foo_bar", + expectedValue: "value", + expectedError: nil, + }, + { + name: "valid key starting with underscore", + line: "_FOO=value", + expectedKey: "_FOO", + expectedValue: "value", + expectedError: nil, + }, + { + name: "valid key with digits", + line: "FOO_123_BAR=value", + expectedKey: "FOO_123_BAR", + expectedValue: "value", + expectedError: nil, + }, + { + name: "valid key with hash in value", + line: "FOO_123_BAR=\"value#withhash\"", + expectedKey: "FOO_123_BAR", + expectedValue: "value#withhash", + expectedError: nil, + }, + { + name: "valid key with empty space / hash in value", + line: "FOO_123_BAR=\"value #withhash\"", + expectedKey: "FOO_123_BAR", + expectedValue: "value #withhash", + expectedError: nil, + }, + { + name: "valid key with uneven quotes", + line: "FOO_123_BAR=\"value", + expectedKey: "FOO_123_BAR", + expectedValue: "", + expectedError: fmt.Errorf("invalid line: FOO_123_BAR=\"value (uneven quotes)"), + }, + { + name: "empty quoted value", + line: "KEY=\"\"", + expectedKey: "KEY", + expectedValue: "", + expectedError: nil, + }, + { + name: "single quoted value", + line: "KEY='value'", + expectedKey: "KEY", + expectedValue: "value", + expectedError: nil, + }, + { + name: "single quotes with hash", + line: "KEY='value # with hash'", + expectedKey: "KEY", + expectedValue: "value # with hash", + expectedError: nil, + }, + { + name: "uneven single quotes", + line: "KEY='value", + expectedKey: "KEY", + expectedValue: "", + expectedError: fmt.Errorf("invalid line: KEY='value (uneven quotes)"), + }, + { + name: "value with equals sign", + line: "KEY=value=another", + expectedKey: "KEY", + expectedValue: "value=another", + expectedError: nil, + }, + { + name: "quoted value with leading/trailing spaces", + line: "KEY=\" value \"", + expectedKey: "KEY", + expectedValue: " value ", + expectedError: nil, + }, + { + name: "empty unquoted value", + line: "KEY=", + expectedKey: "KEY", + expectedValue: "", + expectedError: nil, + }, + { + name: "quoted value with inline comment after", + line: "KEY=\"value\" # comment", + expectedKey: "KEY", + expectedValue: "value", + expectedError: nil, + }, + { + name: "mixed quotes should fail", + line: "KEY=\"value'", + expectedKey: "KEY", + expectedValue: "", + expectedError: fmt.Errorf("invalid line: KEY=\"value' (uneven quotes)"), + }, + { + name: "value with newline in quotes", + line: "KEY=\"line1\\nline2\"", + expectedKey: "KEY", + expectedValue: "line1\\nline2", + expectedError: nil, + }, + } + + for _, example := range examples { + t.Run(example.name, func(t *testing.T) { + key, value, err := parseEnvLine(example.line) + if key != example.expectedKey { + t.Errorf("Expected key '%s' but got '%s'", example.expectedKey, key) + } + if value != example.expectedValue { + t.Errorf("Expected value '%s' but got '%s'", example.expectedValue, value) + } + if (err == nil && example.expectedError != nil) || (err != nil && example.expectedError == nil) || (err != nil && example.expectedError != nil && err.Error() != example.expectedError.Error()) { + t.Errorf("Expected error '%v' but got '%v'", example.expectedError, err) + } + }) + } +} + +func TestEnvFileResolve(t *testing.T) { + // Create examples of env files and expected contents / errors + examples := []struct { + name string + fileContents string + expectedItems map[string]string + expectedErrors []string + expectedSecretsRequested []string + }{ + { + name: "empty env file", + fileContents: "", + expectedItems: map[string]string{}, + expectedErrors: []string{}, + expectedSecretsRequested: []string{}, + }, + { + name: "simple env var", + fileContents: "FOO=bar", + expectedItems: map[string]string{"FOO": "bar"}, + expectedErrors: []string{}, + expectedSecretsRequested: []string{}, + }, + { + name: "simple env var with empty lines", + fileContents: "\nFOO=bar\n", + expectedItems: map[string]string{"FOO": "bar"}, + expectedErrors: []string{}, + expectedSecretsRequested: []string{}, + }, + { + name: "simple env var with comments", + fileContents: "#This is a comment\nFOO=bar # This is another comment\n", + expectedItems: map[string]string{"FOO": "bar"}, + expectedErrors: []string{}, + expectedSecretsRequested: []string{}, + }, + + { + name: "simple env var in sm", + fileContents: "FOO=sm://path/to/secret", + expectedItems: map[string]string{"FOO": "resolved-value-for-sm://path/to/secret"}, + expectedErrors: []string{}, + expectedSecretsRequested: []string{"path/to/secret"}, + }, + { + name: "simple env var not found in sm", + fileContents: "FOO=sm://path/to/not-found", + expectedItems: map[string]string{}, + expectedErrors: []string{"secret not found: sm://path/to/not-found"}, + expectedSecretsRequested: []string{"path/to/not-found"}, + }, + { + name: "duplicate env var should return error", + fileContents: "FOO=bar\nFOO=baz", + expectedItems: map[string]string{"FOO": "bar"}, + expectedErrors: []string{"duplicate key 'FOO' found in env file, duplicate keys are not supported"}, + expectedSecretsRequested: []string{}, + }, + { + name: "key with dash should return validation error", + fileContents: "foo-bar=value1", + expectedItems: map[string]string{}, + expectedErrors: []string{"invalid key 'foo-bar': environment variable names must contain only letters, digits, and underscores, and must start with a letter or underscore"}, + expectedSecretsRequested: []string{}, + }, + { + name: "duplicate keys with same format should error", + fileContents: "MY_KEY=value1\nMY_KEY=value2", + expectedItems: map[string]string{"MY_KEY": "value1"}, + expectedErrors: []string{"duplicate key 'MY_KEY' found in env file, duplicate keys are not supported"}, + expectedSecretsRequested: []string{}, + }, + { + name: "multiple env vars pointing to same secret should dedupe API calls", + fileContents: `FOO=sm://shared/secret +BAR=sm://shared/secret +BAZ=sm://other/secret +QUX=sm://shared/secret`, + expectedItems: map[string]string{ + "FOO": "resolved-value-for-sm://shared/secret", + "BAR": "resolved-value-for-sm://shared/secret", + "BAZ": "resolved-value-for-sm://other/secret", + "QUX": "resolved-value-for-sm://shared/secret", + }, + expectedErrors: []string{}, + // Should only request each unique secret path once + expectedSecretsRequested: []string{"shared/secret", "other/secret"}, + }, + { + name: "keys are preserved exactly without normalization", + fileContents: "lowercase_var=value1\nMIXED_Case_Var=value2\nUPPERCASE_VAR=value3", + expectedItems: map[string]string{"lowercase_var": "value1", "MIXED_Case_Var": "value2", "UPPERCASE_VAR": "value3"}, + expectedErrors: []string{}, + expectedSecretsRequested: []string{}, + }, + } + + requestedSecrets := []string{} + mockSecretsManagerConnector := &connectortesting.MockSecretsConnector{ + GetSecretsFunc: func(keys []string) (map[string]string, []error) { + requestedSecrets = append(requestedSecrets, keys...) + secrets := make(map[string]string) + errors := []error{} + + for _, key := range keys { + if strings.Contains(key, "not-found") { + errors = append(errors, fmt.Errorf("secret not found: sm://%s", key)) + } else { + // Note: keys come without the "sm://" prefix as the resolver strips it + secrets[key] = "resolved-value-for-sm://" + key + } + } + return secrets, errors + }, + } + + for _, example := range examples { + t.Run(example.name, func(t *testing.T) { + requestedSecrets = []string{} // reset before each test + // Create string readers and pass them to resolve function so tests are faster + result := &Result{} + envFileResolver := &EnvFileResolver{ + connector: mockSecretsManagerConnector, + } + envFileResolver.resolve(strings.NewReader(example.fileContents), result) + + // Check that the number of items matches + if len(result.Items) != len(example.expectedItems) { + t.Errorf("Expected %d items but got %d", len(example.expectedItems), len(result.Items)) + } + + // Check that expected items are present in the result + for key, value := range example.expectedItems { + actualValue, ok := result.Items[key] + if !ok { + t.Errorf("Expected item %s not found in result", key) + } else if actualValue != value { + t.Errorf("Expected item %s to have value %s but got %s", key, value, actualValue) + } + } + + // Check that non expected items are not present in the result + for key := range result.Items { + if _, ok := example.expectedItems[key]; !ok { + t.Errorf("Unexpected item %s found in result", key) + } + } + + // Check that the number of errors matches + if len(result.Errors) != len(example.expectedErrors) { + t.Errorf("Expected %d errors but got %d", len(example.expectedErrors), len(result.Errors)) + } + + // Check that expected errors are present in the result + for _, expectedError := range example.expectedErrors { + found := false + for _, err := range result.Errors { + if err.Error() == expectedError { + found = true + break + } + } + if !found { + t.Errorf("Expected error '%s' not found in result errors", expectedError) + } + } + + // Check that the secrets requested match expectations + if len(requestedSecrets) != len(example.expectedSecretsRequested) { + t.Errorf("Expected %d secrets to be requested but got %d: %v", len(example.expectedSecretsRequested), len(requestedSecrets), requestedSecrets) + } + + // Create maps for easier comparison (order doesn't matter) + requestedMap := make(map[string]bool) + for _, secret := range requestedSecrets { + requestedMap[secret] = true + } + expectedMap := make(map[string]bool) + for _, secret := range example.expectedSecretsRequested { + expectedMap[secret] = true + } + + // Check that all expected secrets were requested + for secret := range expectedMap { + if !requestedMap[secret] { + t.Errorf("Expected secret '%s' to be requested but it wasn't", secret) + } + } + + // Check that no unexpected secrets were requested + for secret := range requestedMap { + if !expectedMap[secret] { + t.Errorf("Unexpected secret '%s' was requested", secret) + } + } + }) + } + +} + +// Using actual tmp files to test the full feature + +const envFileContents = `# This is a comment +FOO=bar +BAZ=sm://path/to/baz +TEST=12345 # Inline comment +` + +func TestEnvFileIntegrationTest(t *testing.T) { + // Create a temporary .env file + tmpFile, err := os.CreateTemp("", "envfile-test-*.env") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + _, err = tmpFile.WriteString(envFileContents) + if err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create the source configuration + source := &config.Source{ + URL: &url.URL{ + Scheme: "sm", + Host: "us-east-1", + Path: "/", + }, + } + + // Create the SecretsManagerConnector with fake secrets for testing + secretsManagerConnector := connectortesting.NewSecretsManagerConnectorWithFakeSecrets( + map[string]string{ + "path/to/baz": "secret-from-aws", + }, + source, + ) + + // Create the real EnvFileResolver with the real connector + envFileResolver := NewEnvFileResolver(secretsManagerConnector) + + // Create file source for the resolver + fileSource := &config.Source{ + URL: &url.URL{ + Scheme: "file", + Path: tmpFile.Name(), + }, + } + + // Resolve the env file (this tests the full integration) + result := envFileResolver.Resolve(fileSource) + + // Verify results + expectedItems := map[string]string{ + "FOO": "bar", + "BAZ": "secret-from-aws", + "TEST": "12345", + } + + if len(result.Items) != len(expectedItems) { + t.Errorf("Expected %d items but got %d", len(expectedItems), len(result.Items)) + } + + for key, value := range expectedItems { + actualValue, ok := result.Items[key] + if !ok { + t.Errorf("Expected item %s not found in result", key) + } else if actualValue != value { + t.Errorf("Expected item %s to have value %s but got %s", key, value, actualValue) + } + } + + for key := range result.Items { + if _, ok := expectedItems[key]; !ok { + t.Errorf("Unexpected item %s found in result", key) + } + } + + // Should have no errors for this happy path test + if len(result.Errors) != 0 { + t.Errorf("Expected 0 errors but got %d: %v", len(result.Errors), result.Errors) + } +} + +// TestEnvFileIntegrationTestWithMissingSecret tests the integration when a secret is not found in AWS +func TestEnvFileIntegrationTestWithMissingSecret(t *testing.T) { + // Create a temporary .env file with a reference to a non-existent secret + tmpFile, err := os.CreateTemp("", "envfile-test-*.env") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + testContent := `FOO=bar +BAZ=sm://path/to/missing-secret +TEST=12345 +` + _, err = tmpFile.WriteString(testContent) + if err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create the source configuration + source := &config.Source{ + URL: &url.URL{ + Scheme: "sm", + Host: "us-east-1", + Path: "/", + }, + } + + // Create the SecretsManagerConnector with no fake secrets for testing + secretsManagerConnector := connectortesting.NewSecretsManagerConnectorWithFakeSecrets( + map[string]string{}, + source, + ) + + // Create the real EnvFileResolver with the real connector + envFileResolver := NewEnvFileResolver(secretsManagerConnector) + + // Create file source for the resolver + fileSource := &config.Source{ + URL: &url.URL{ + Scheme: "file", + Path: tmpFile.Name(), + }, + } + + // Resolve the env file + result := envFileResolver.Resolve(fileSource) + + // Verify that only non-secret items are present + expectedItems := map[string]string{ + "FOO": "bar", + "TEST": "12345", + } + + if len(result.Items) != len(expectedItems) { + t.Errorf("Expected %d items but got %d", len(expectedItems), len(result.Items)) + } + + for key, value := range expectedItems { + actualValue, ok := result.Items[key] + if !ok { + t.Errorf("Expected item %s not found in result", key) + } else if actualValue != value { + t.Errorf("Expected item %s to have value %s but got %s", key, value, actualValue) + } + } + + // Should have one error for the missing secret + if len(result.Errors) != 1 { + t.Errorf("Expected 1 error but got %d", len(result.Errors)) + } else { + errorMsg := result.Errors[0].Error() + if !strings.Contains(errorMsg, "path/to/missing-secret") && !strings.Contains(errorMsg, "not found") { + t.Errorf("Expected error message to mention missing secret, got: %s", errorMsg) + } + } +} diff --git a/pkg/resolvers/resolvers.go b/pkg/resolvers/resolvers.go index af89865..b7337af 100644 --- a/pkg/resolvers/resolvers.go +++ b/pkg/resolvers/resolvers.go @@ -34,6 +34,16 @@ func (r *Result) AppendItem(key, value string) { r.Items[key] = value } +// AppendItemExact adds an item to the internal Items map without any key normalization. +// Use this when you need to preserve the exact key as provided (e.g., from env files). +func (r *Result) AppendItemExact(key, value string) { + // Initialize if we have to + if r.Items == nil { + r.Items = map[string]string{} + } + r.Items[key] = value +} + // AppendItems adds a map of items to our internal items store func (r *Result) AppendItems(items map[string]string) { // Initialize if we have to @@ -71,6 +81,19 @@ func (r *Result) LenItems() int { // ResolveSource will resolve a config.Source to a Result object func ResolveSource(source *config.Source) *Result { + if source == nil { + return &Result{ + Source: nil, + Errors: []error{fmt.Errorf("resolvers.ResolveSource: source must not be nil")}, + } + } + + if source.URL == nil { + return &Result{ + Source: source, + Errors: []error{fmt.Errorf("resolvers.ResolveSource: source.URL must not be nil")}, + } + } sourceURL := source.URL var s Resolver switch sourceURL.Scheme { @@ -88,6 +111,12 @@ func ResolveSource(source *config.Source) *Result { return &Result{Source: source, Errors: []error{err}} } s = NewManifestResolver(connector) + case "file": + connector, err := connectors.NewSecretsManagerConnector(source) + if err != nil { + return &Result{Source: source, Errors: []error{err}} + } + s = NewEnvFileResolver(connector) default: return &Result{Source: source, Errors: []error{fmt.Errorf("No resolver found for scheme %s", sourceURL.Scheme)}} } diff --git a/pkg/resolvers/resolvers_test.go b/pkg/resolvers/resolvers_test.go index cef2705..fcf784b 100644 --- a/pkg/resolvers/resolvers_test.go +++ b/pkg/resolvers/resolvers_test.go @@ -131,20 +131,25 @@ func TestResolveSource(t *testing.T) { t.Error("Expected at least one error in Errors slice") } - // Test with each valid scheme (will error due to no AWS, but scheme routing works) - schemes := []string{"s3", "sm", "manifest"} + // Test with each valid scheme (will error due to no AWS/missing file, but scheme routing works) + schemes := []string{"s3", "sm", "manifest", "file"} for _, scheme := range schemes { testURL, _ := url.Parse(scheme + "://test/path") testSource := &config.Source{URL: testURL} result := ResolveSource(testSource) - // Result should exist (even if it has errors due to missing AWS resources) + // Result should exist (even if it has errors due to missing AWS resources or files) if result == nil { t.Errorf("Expected result for scheme %s, got nil", scheme) } if result.Source != testSource { t.Errorf("Expected result.Source to match input source for scheme %s", scheme) } + + // For file scheme, we expect an error since the file doesn't exist + if scheme == "file" && !result.HasErrors() { + t.Errorf("Expected file scheme to error for non-existent file") + } } }