Add SerializationFeature.ORDER_SET_ENTRIES_BY_ELEMENTS for #3166

src/main/java/com/fasterxml/jackson/databind/SerializationFeature.java

@@ -461,6 +461,38 @@ public enum SerializationFeature implements ConfigFeature
      */
     FAIL_ON_ORDER_MAP_BY_INCOMPARABLE_KEY(true),
 
+    /**
+     * Feature that determines whether {@link java.util.Set} entries are first
+     * sorted before serialization or not: if enabled, additional sorting step
+     * is performed if necessary (not necessary for
+     * {@link java.util.SortedSet}s), if disabled, no additional sorting is needed.
+     *<p>
+     * Note that sorting requires set elements to implement {@link java.lang.Comparable};
+     * behavior when encountering non-Comparable elements is controlled by
+     * {@link #FAIL_ON_ORDER_SET_BY_INCOMPARABLE_ELEMENT}.
+     *<p>
+     * Feature is disabled by default.
+     *
+     * @since 2.22
+     */
+    ORDER_SET_ENTRIES_BY_ELEMENTS(false),
+
+    /**
+     * Feature that determines whether to fail when attempting to sort
+     * {@link java.util.Set} entries with non-Comparable or mutually incomparable elements.
+     *<p>
+     * If enabled, will throw an exception when set elements cannot be sorted.
+     * If disabled, will silently skip sorting and use the original iteration order.
+     *<p>
+     * Note that this feature only applies when set entry ordering is enabled via
+     * {@link #ORDER_SET_ENTRIES_BY_ELEMENTS}.
+     *<p>
+     * Feature is enabled by default.
+     *
+     * @since 2.22
+     */
+    FAIL_ON_ORDER_SET_BY_INCOMPARABLE_ELEMENT(true),
+
     /*
     /******************************************************
     /* Other

src/main/java/com/fasterxml/jackson/databind/ser/impl/StringCollectionSerializer.java

@@ -97,10 +97,19 @@ private final void serializeContents(Collection<String> value, JsonGenerator g,
             SerializerProvider provider)
         throws IOException
     {
-        int i = 0;
+        // [databind#3166]: sort Set<String> if feature enabled
+        Collection<String> toSerialize = value;
+        if (value instanceof Set<?> && !(value instanceof SortedSet<?>)) {
+            if (provider.isEnabled(SerializationFeature.ORDER_SET_ENTRIES_BY_ELEMENTS)) {
+                List<String> sorted = new ArrayList<>(value);
+                sorted.sort(Comparator.nullsLast(Comparator.naturalOrder()));
+                toSerialize = sorted;
+            }
+        }
 
+        int i = 0;
         try {
-            for (String str : value) {
+            for (String str : toSerialize) {
                 if (str == null) {
                     provider.defaultSerializeNull(g);
                 } else {

src/main/java/com/fasterxml/jackson/databind/ser/std/CollectionSerializer.java

@@ -1,9 +1,7 @@
 package com.fasterxml.jackson.databind.ser.std;
 
 import java.io.IOException;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.Iterator;
+import java.util.*;
 
 import com.fasterxml.jackson.core.*;
 import com.fasterxml.jackson.databind.BeanProperty;
@@ -115,12 +113,21 @@ public final void serialize(Collection<?> value, JsonGenerator g, SerializerProv
     @Override
     public void serializeContents(Collection<?> value, JsonGenerator g, SerializerProvider provider) throws IOException
     {
+        // [databind#3166]: sort Set elements if feature enabled
+        Collection<?> toSerialize = value;
+        if (value instanceof Set<?> && !(value instanceof SortedSet<?>)
+                && !(value instanceof EnumSet<?>)) {
+            if (provider.isEnabled(SerializationFeature.ORDER_SET_ENTRIES_BY_ELEMENTS)) {
+                toSerialize = _orderElements(value, provider);
+            }
+        }
+
         g.assignCurrentValue(value);
         if (_elementSerializer != null) {
-            serializeContentsUsing(value, g, provider, _elementSerializer);
+            serializeContentsUsing(toSerialize, g, provider, _elementSerializer);
             return;
         }
-        Iterator<?> it = value.iterator();
+        Iterator<?> it = toSerialize.iterator();
         if (!it.hasNext()) {
             return;
         }
@@ -160,6 +167,43 @@ public void serializeContents(Collection<?> value, JsonGenerator g, SerializerPr
         }
     }
 
+    /**
+     * Helper method to sort Set elements for deterministic serialization.
+     *
+     * @since 2.22
+     */
+    @SuppressWarnings("unchecked")
+    protected Collection<?> _orderElements(Collection<?> input,
+            SerializerProvider provider) throws IOException
+    {
+        if (input instanceof SortedSet<?>) {
+            return input;
+        }
+        if (input instanceof EnumSet<?>) {
+            return input;
+        }
+        if (input.isEmpty()) {
+            return input;
+        }
+        try {
+            List<Object> sorted = new ArrayList<>(input);
+            sorted.sort((Comparator<Object>)(Comparator<?>)
+                    Comparator.nullsLast(Comparator.naturalOrder()));
+            return sorted;
+        } catch (ClassCastException e) {
+            if (!provider.isEnabled(
+                    SerializationFeature.FAIL_ON_ORDER_SET_BY_INCOMPARABLE_ELEMENT)) {
+                return input;
+            }
+            provider.reportBadDefinition(input.getClass(),
+                "Cannot order Set entries: elements are not mutually "
+                + "Comparable, consider disabling "
+                + "`SerializationFeature.FAIL_ON_ORDER_SET_BY_INCOMPARABLE_ELEMENT`"
+                + " to simply skip sorting");
+            return input; // unreachable, reportBadDefinition throws
+        }
+    }
+
     public void serializeContentsUsing(Collection<?> value, JsonGenerator g, SerializerProvider provider,
             JsonSerializer<Object> ser) throws IOException
     {