Add application listener to locate property sources during bootstrap

docs/src/main/asciidoc/_configprops.adoc

@@ -4,6 +4,7 @@
 |spring.cloud.compatibility-verifier.compatible-boot-versions |  | Default accepted versions for the Spring Boot dependency. You can set {@code x} for the patch version if you don't want to specify a concrete value. Example: {@code 3.4.x}
 |spring.cloud.compatibility-verifier.enabled | `+++false+++` | Enables creation of Spring Cloud compatibility verification.
 |spring.cloud.config.allow-override | `+++true+++` | Flag to indicate that {@link #isOverrideSystemProperties() systemPropertiesOverride} can be used. Set to false to prevent users from changing the default accidentally. Default true.
+|spring.cloud.config.initialize-on-context-refresh | `+++false+++` | Flag to initialize bootstrap configuration on context refresh event. Default false.
 |spring.cloud.config.override-none | `+++false+++` | Flag to indicate that when {@link #setAllowOverride(boolean) allowOverride} is true, external properties should take lowest priority and should not override any existing property sources (including local config files). Default false.
 |spring.cloud.config.override-system-properties | `+++true+++` | Flag to indicate that the external properties should override system properties. Default true.
 |spring.cloud.decrypt-environment-post-processor.enabled | `+++true+++` | Enable the DecryptEnvironmentPostProcessor.

docs/src/main/asciidoc/spring-cloud-commons.adoc

@@ -54,6 +54,10 @@ The additional property sources are:
 An example would be properties from the Spring Cloud Config Server.
 See "`<<customizing-bootstrap-property-sources>>`" for how to customize the contents of this property source.
 
+NOTE:  Prior to Spring Cloud 2021.0.7 `PropertySourceLocators` (including the ones for Spring Cloud Config) were run during
+the main application context and not in the Bootstrap context.  You can force `PropertySourceLocators` to be run during the
+Bootstrap context by setting `spring.cloud.config.initialize-on-context-refresh=true` in `bootstrap.[properties | yaml]`.
+
 * "`applicationConfig: [classpath:bootstrap.yml]`" (and related files if Spring profiles are active): If you have a `bootstrap.yml` (or `.properties`), those properties are used to configure the bootstrap context.
 Then they get added to the child context when its parent is set.
 They have lower precedence than the `application.yml` (or `.properties`) and any other property sources that are added to the child as a normal part of the process of creating a Spring Boot application.
@@ -146,6 +150,12 @@ org.springframework.cloud.bootstrap.BootstrapConfiguration=sample.custom.CustomP
 ----
 ====
 
+As of Spring Cloud 2021.0.8, Spring Cloud will now call `PropertySourceLocators` twice.  The first fetch
+will retrieve any property sources without any profiles.  These property sources will have the opportunity to
+activate profiles using `spring.profiles.active`.  After the main application context starts `PropertySourceLocators`
+will be called a second time, this time with any active profiles allowing `PropertySourceLocators` to locate
+any additional `PropertySources` with profiles.
+
 === Logging Configuration
 
 If you use Spring Boot to configure log settings, you should place this configuration in `bootstrap.[yml | properties]` if you would like it to apply to all events.

spring-cloud-context/src/main/java/org/springframework/cloud/bootstrap/BootstrapApplicationListener.java

@@ -294,6 +294,13 @@ private void apply(ConfigurableApplicationContext context, SpringApplication app
 		target.addAll(getOrderedBeansOfType(context, ApplicationContextInitializer.class));
 		application.setInitializers(target);
 		addBootstrapDecryptInitializer(application);
+
+		// Get the active profiles from the bootstrap context and set them in main
+		// application
+		// environment. This allows any profiles activated during bootstrap to be
+		// activated when
+		// config data runs in the main application context.
+		environment.setActiveProfiles(context.getEnvironment().getActiveProfiles());
 	}
 
 	@SuppressWarnings("unchecked")

spring-cloud-context/src/main/java/org/springframework/cloud/bootstrap/config/PropertySourceBootstrapConfiguration.java

@@ -17,18 +17,20 @@
 package org.springframework.cloud.bootstrap.config;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.context.config.ConfigFileApplicationListener;
+import org.springframework.boot.context.config.Profiles;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.boot.context.properties.bind.Bindable;
 import org.springframework.boot.context.properties.bind.Binder;
@@ -39,8 +41,10 @@
 import org.springframework.cloud.context.environment.EnvironmentChangeEvent;
 import org.springframework.cloud.logging.LoggingRebinder;
 import org.springframework.context.ApplicationContextInitializer;
+import org.springframework.context.ApplicationListener;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.core.Ordered;
 import org.springframework.core.annotation.AnnotationAwareOrderComparator;
 import org.springframework.core.env.AbstractEnvironment;
@@ -52,6 +56,7 @@
 import org.springframework.core.env.PropertySource;
 import org.springframework.util.StringUtils;
 
+import static org.springframework.cloud.bootstrap.encrypt.AbstractEnvironmentDecrypt.DECRYPTED_PROPERTY_SOURCE_NAME;
 import static org.springframework.core.env.StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME;
 
 /**
@@ -60,8 +65,8 @@
  */
 @Configuration(proxyBeanMethods = false)
 @EnableConfigurationProperties(PropertySourceBootstrapProperties.class)
-public class PropertySourceBootstrapConfiguration
-		implements ApplicationContextInitializer<ConfigurableApplicationContext>, Ordered {
+public class PropertySourceBootstrapConfiguration implements ApplicationListener<ContextRefreshedEvent>,
+		ApplicationContextInitializer<ConfigurableApplicationContext>, Ordered {
 
 	/**
 	 * Bootstrap property source name.
@@ -76,6 +81,9 @@ public class PropertySourceBootstrapConfiguration
 	@Autowired(required = false)
 	private List<PropertySourceLocator> propertySourceLocators = new ArrayList<>();
 
+	@Autowired
+	private PropertySourceBootstrapProperties bootstrapProperties;
+
 	@Override
 	public int getOrder() {
 		return this.order;
@@ -85,8 +93,25 @@ public void setPropertySourceLocators(Collection<PropertySourceLocator> property
 		this.propertySourceLocators = new ArrayList<>(propertySourceLocators);
 	}
 
+	/*
+	 * The ApplicationListener is called when the main application context is initialized.
+	 * This will be called after the ApplicationListener ContextRefreshedEvent is fired
+	 * during the bootstrap phase. This method is also what added PropertySources prior to
+	 * Spring Cloud 2021.0.7, this is why it will be called when
+	 * spring.cloud.config.initialize-on-context-refresh is false. When
+	 * spring.cloud.config.initialize-on-context-refresh is true this method provides a
+	 * "second fetch" of configuration data to fetch any additional configuration data
+	 * from profiles that have been activated.
+	 */
 	@Override
 	public void initialize(ConfigurableApplicationContext applicationContext) {
+		if (!bootstrapProperties.isInitializeOnContextRefresh() || !applicationContext.getEnvironment()
+				.getPropertySources().contains(BootstrapApplicationListener.BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
+			doInitialize(applicationContext);
+		}
+	}
+
+	private void doInitialize(ConfigurableApplicationContext applicationContext) {
 		List<PropertySource<?>> composite = new ArrayList<>();
 		AnnotationAwareOrderComparator.sort(this.propertySourceLocators);
 		boolean empty = true;
@@ -122,7 +147,7 @@ public void initialize(ConfigurableApplicationContext applicationContext) {
 			insertPropertySources(propertySources, composite);
 			reinitializeLoggingSystem(environment, logConfig, logFile);
 			setLogLevels(applicationContext, environment);
-			handleIncludedProfiles(environment);
+			handleProfiles(environment);
 		}
 	}
 
@@ -172,7 +197,12 @@ private void insertPropertySources(MutablePropertySources propertySources, List<
 		if (!remoteProperties.isAllowOverride()
 				|| (!remoteProperties.isOverrideNone() && remoteProperties.isOverrideSystemProperties())) {
 			for (PropertySource<?> p : reversedComposite) {
-				propertySources.addFirst(p);
+				if (propertySources.contains(DECRYPTED_PROPERTY_SOURCE_NAME)) {
+					propertySources.addAfter(DECRYPTED_PROPERTY_SOURCE_NAME, p);
+				}
+				else {
+					propertySources.addFirst(p);
+				}
 			}
 			return;
 		}
@@ -210,43 +240,98 @@ private Environment environment(MutablePropertySources incoming) {
 		return environment;
 	}
 
-	private void handleIncludedProfiles(ConfigurableEnvironment environment) {
+	private void handleProfiles(ConfigurableEnvironment environment) {
+		if (bootstrapProperties.isInitializeOnContextRefresh() && !environment.getPropertySources()
+				.contains(BootstrapApplicationListener.BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
+			// In the case that spring.cloud.config.initialize-on-context-refresh is true
+			// this method will
+			// be called during the bootstrap phase and the main application startup. We
+			// only manipulate the environment profiles in the bootstrap phase as we are
+			// fetching
+			// any additional profile specific configuration when this method would be
+			// called during the
+			// main application startup, and it is not valid to activate profiles in
+			// profile specific
+			// configuration properties, so we should not run this method then.
+			return;
+		}
 		Set<String> includeProfiles = new TreeSet<>();
+		List<String> activeProfiles = new ArrayList<>();
+
 		for (PropertySource<?> propertySource : environment.getPropertySources()) {
-			addIncludedProfilesTo(includeProfiles, propertySource);
+			addIncludedProfilesTo(includeProfiles, propertySource, environment);
+			addActiveProfilesTo(activeProfiles, propertySource, environment);
 		}
-		List<String> activeProfiles = new ArrayList<>();
-		Collections.addAll(activeProfiles, environment.getActiveProfiles());
 
 		// If it's already accepted we assume the order was set intentionally
 		includeProfiles.removeAll(activeProfiles);
-		if (includeProfiles.isEmpty()) {
-			return;
-		}
 		// Prepend each added profile (last wins in a property key clash)
 		for (String profile : includeProfiles) {
 			activeProfiles.add(0, profile);
 		}
+		List<String> activeProfilesFromEnvironment = Arrays.stream(environment.getActiveProfiles())
+				.collect(Collectors.toList());
+		if (!activeProfiles.containsAll(activeProfilesFromEnvironment)) {
+			activeProfiles.addAll(activeProfilesFromEnvironment);
+
+		}
 		environment.setActiveProfiles(activeProfiles.toArray(new String[activeProfiles.size()]));
 	}
 
-	private Set<String> addIncludedProfilesTo(Set<String> profiles, PropertySource<?> propertySource) {
+	private Set<String> addIncludedProfilesTo(Set<String> profiles, PropertySource<?> propertySource,
+			ConfigurableEnvironment environment) {
+		return addProfilesTo(profiles, propertySource, Profiles.INCLUDE_PROFILES_PROPERTY_NAME, environment);
+	}
+
+	private List<String> addActiveProfilesTo(List<String> profiles, PropertySource<?> propertySource,
+			ConfigurableEnvironment environment) {
+		return addProfilesTo(profiles, propertySource, AbstractEnvironment.ACTIVE_PROFILES_PROPERTY_NAME, environment);
+	}
+
+	private <T extends Collection<String>> T addProfilesTo(T profiles, PropertySource<?> propertySource,
+			String property, ConfigurableEnvironment environment) {
 		if (propertySource instanceof CompositePropertySource) {
 			for (PropertySource<?> nestedPropertySource : ((CompositePropertySource) propertySource)
 					.getPropertySources()) {
-				addIncludedProfilesTo(profiles, nestedPropertySource);
+				addProfilesTo(profiles, nestedPropertySource, property, environment);
 			}
 		}
 		else {
-			Collections.addAll(profiles, getProfilesForValue(
-					propertySource.getProperty(ConfigFileApplicationListener.INCLUDE_PROFILES_PROPERTY)));
+			Collections.addAll(profiles, getProfilesForValue(propertySource.getProperty(property), environment));
 		}
 		return profiles;
 	}
 
-	private String[] getProfilesForValue(Object property) {
+	private String[] getProfilesForValue(Object property, ConfigurableEnvironment environment) {
 		final String value = (property == null ? null : property.toString());
-		return property == null ? new String[0] : StringUtils.tokenizeToStringArray(value, ",");
+		return property == null ? new String[0] : resolvePlaceholdersInProfiles(value, environment);
+	}
+
+	private String[] resolvePlaceholdersInProfiles(String profiles, ConfigurableEnvironment environment) {
+		return Arrays.stream(StringUtils.tokenizeToStringArray(profiles, ",")).map(s -> {
+			if (s.startsWith("${") && s.endsWith("}")) {
+				return environment.resolvePlaceholders(s);
+			}
+			else {
+				return s;
+			}
+		}).toArray(String[]::new);
+	}
+
+	/*
+	 * The ConextRefreshedEvent gets called at the end of the boostrap phase after config
+	 * data is loaded during bootstrap. This will run and do an "initial fetch" of
+	 * configuration data during bootstrap but before the main applicaiton context starts.
+	 */
+	@Override
+	public void onApplicationEvent(ContextRefreshedEvent event) {
+		if (bootstrapProperties.isInitializeOnContextRefresh()
+				&& event.getApplicationContext() instanceof ConfigurableApplicationContext) {
+			if (((ConfigurableApplicationContext) event.getApplicationContext()).getEnvironment().getPropertySources()
+					.contains(BootstrapApplicationListener.BOOTSTRAP_PROPERTY_SOURCE_NAME)) {
+				doInitialize((ConfigurableApplicationContext) event.getApplicationContext());
+			}
+		}
 	}
 
 }

spring-cloud-context/src/main/java/org/springframework/cloud/bootstrap/config/PropertySourceBootstrapProperties.java

@@ -46,6 +46,19 @@ public class PropertySourceBootstrapProperties {
 	 */
 	private boolean overrideNone = false;
 
+	/**
+	 * Flag to initialize bootstrap configuration on context refresh event. Default false.
+	 */
+	private boolean initializeOnContextRefresh = false;
+
+	public boolean isInitializeOnContextRefresh() {
+		return initializeOnContextRefresh;
+	}
+
+	public void setInitializeOnContextRefresh(boolean initializeOnContextRefresh) {
+		this.initializeOnContextRefresh = initializeOnContextRefresh;
+	}
+
 	public boolean isOverrideNone() {
 		return this.overrideNone;
 	}