Skip to content

Commit e91b47c

Browse files
heemin32heemin32
authored
Add default value in denylist (opensearch-project#583)
Signed-off-by: Heemin Kim <heemin@amazon.com>
1 parent 3b74ae4 commit e91b47c

6 files changed

Lines changed: 155 additions & 5 deletions

File tree

‎src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import java.net.MalformedURLException;
99
import java.net.URISyntaxException;
1010
import java.net.URL;
11-
import java.util.Collections;
11+
import java.util.Arrays;
1212
import java.util.List;
1313
import java.util.function.Function;
1414

@@ -77,10 +77,34 @@ public class Ip2GeoSettings {
7777

7878
/**
7979
* A list of CIDR which will be blocked to be used as datasource endpoint
80+
* Private network addresses will be blocked as default
8081
*/
8182
public static final Setting<List<String>> DATASOURCE_ENDPOINT_DENYLIST = Setting.listSetting(
8283
"plugins.geospatial.ip2geo.datasource.endpoint.denylist",
83-
Collections.emptyList(),
84+
Arrays.asList(
85+
"127.0.0.0/8",
86+
"169.254.0.0/16",
87+
"10.0.0.0/8",
88+
"172.16.0.0/12",
89+
"192.168.0.0/16",
90+
"0.0.0.0/8",
91+
"100.64.0.0/10",
92+
"192.0.0.0/24",
93+
"192.0.2.0/24",
94+
"198.18.0.0/15",
95+
"192.88.99.0/24",
96+
"198.51.100.0/24",
97+
"203.0.113.0/24",
98+
"224.0.0.0/4",
99+
"240.0.0.0/4",
100+
"255.255.255.255/32",
101+
"::1/128",
102+
"fe80::/10",
103+
"fc00::/7",
104+
"::/128",
105+
"2001:db8::/32",
106+
"ff00::/8"
107+
),
84108
Function.identity(),
85109
Setting.Property.NodeScope,
86110
Setting.Property.Dynamic

‎src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,20 @@ protected Map<String, Object> simulatePipeline(final String name, List<Object> d
193193
return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map();
194194
}
195195

196+
protected Response updateClusterSetting(final Map<String, Object> properties) throws IOException {
197+
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
198+
builder.startObject("transient");
199+
for (Map.Entry<String, Object> config : properties.entrySet()) {
200+
builder.field(config.getKey(), config.getValue());
201+
}
202+
builder.endObject();
203+
builder.endObject();
204+
205+
Request request = new Request(PUT, "/_cluster/settings");
206+
request.setJsonEntity(builder.toString());
207+
return client().performRequest(request);
208+
}
209+
196210
protected static void createIndex(String name, Settings settings, Map<String, String> fieldMap) throws IOException {
197211
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject(MAPPING_PROPERTIES_KEY);
198212
for (Map.Entry<String, String> entry : fieldMap.entrySet()) {

‎src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55

66
package org.opensearch.geospatial.ip2geo;
77

8+
import static org.mockito.ArgumentMatchers.anyString;
89
import static org.mockito.Mockito.spy;
910
import static org.mockito.Mockito.when;
1011

1112
import java.io.File;
13+
import java.net.URL;
1214
import java.nio.file.Paths;
1315
import java.time.Instant;
1416
import java.time.temporal.ChronoUnit;
@@ -99,6 +101,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase {
99101
protected Ip2GeoProcessorDao ip2GeoProcessorDao;
100102
@Mock
101103
protected RoutingTable routingTable;
104+
@Mock
102105
protected URLDenyListChecker urlDenyListChecker;
103106
protected IngestMetadata ingestMetadata;
104107
protected NoOpNodeClient client;
@@ -117,7 +120,7 @@ public void prepareIp2GeoTestCase() {
117120
clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings()));
118121
lockService = new LockService(client, clusterService);
119122
ingestMetadata = new IngestMetadata(Collections.emptyMap());
120-
urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings));
123+
when(urlDenyListChecker.toUrlIfNotInDenyList(anyString())).thenAnswer(i -> new URL(i.getArgument(0)));
121124
when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata);
122125
when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
123126
when(clusterService.getClusterSettings()).thenReturn(clusterSettings);

‎src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import java.io.IOException;
99
import java.time.Duration;
10+
import java.util.Collections;
1011
import java.util.List;
1112
import java.util.Locale;
1213
import java.util.Map;
@@ -16,10 +17,12 @@
1617
import org.junit.AfterClass;
1718
import org.junit.BeforeClass;
1819
import org.opensearch.client.ResponseException;
20+
import org.opensearch.common.settings.Settings;
1921
import org.opensearch.core.rest.RestStatus;
2022
import org.opensearch.geospatial.GeospatialRestTestCase;
2123
import org.opensearch.geospatial.GeospatialTestHelper;
2224
import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer;
25+
import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
2326

2427
public class UpdateDatasourceIT extends GeospatialRestTestCase {
2528
// Use this value in resource name to avoid name conflict among tests
@@ -35,8 +38,50 @@ public static void stop() {
3538
Ip2GeoDataServer.stop();
3639
}
3740

41+
@SneakyThrows
42+
public void testUpdateDatasource_whenPrivateNetwork_thenBlocked() {
43+
// Reset deny list to allow private network access during test
44+
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));
45+
46+
boolean isDatasourceCreated = false;
47+
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
48+
try {
49+
Map<String, Object> datasourceProperties = Map.of(
50+
PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(),
51+
Ip2GeoDataServer.getEndpointCountry()
52+
);
53+
54+
// Create datasource and wait for it to be available
55+
createDatasource(datasourceName, datasourceProperties);
56+
isDatasourceCreated = true;
57+
waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10));
58+
59+
// Revert deny list to its default value and private network ip should be blocked
60+
updateClusterSetting(
61+
Map.of(
62+
Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(),
63+
Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY)
64+
)
65+
);
66+
int updateIntervalInDays = 1;
67+
ResponseException exception = expectThrows(
68+
ResponseException.class,
69+
() -> updateDatasourceEndpoint(datasourceName, "http://127.0.0.1:9200/city/manifest_local.json", updateIntervalInDays)
70+
);
71+
assertEquals(400, exception.getResponse().getStatusLine().getStatusCode());
72+
assertTrue(exception.getMessage().contains("blocked by deny list"));
73+
} finally {
74+
if (isDatasourceCreated) {
75+
deleteDatasource(datasourceName, 3);
76+
}
77+
}
78+
}
79+
3880
@SneakyThrows
3981
public void testUpdateDatasource_whenValidInput_thenUpdated() {
82+
// Reset deny list to allow private network access during test
83+
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));
84+
4085
boolean isDatasourceCreated = false;
4186
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
4287
try {
@@ -65,6 +110,9 @@ public void testUpdateDatasource_whenValidInput_thenUpdated() {
65110

66111
@SneakyThrows
67112
public void testUpdateDatasource_whenIncompatibleFields_thenFails() {
113+
// Reset deny list to allow private network access during test
114+
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));
115+
68116
boolean isDatasourceCreated = false;
69117
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
70118
try {

‎src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55

66
package org.opensearch.geospatial.ip2geo.common;
77

8+
import java.util.Arrays;
9+
import java.util.List;
10+
11+
import org.opensearch.common.settings.Settings;
812
import org.opensearch.test.OpenSearchTestCase;
913

1014
public class Ip2GeoSettingsTests extends OpenSearchTestCase {
@@ -18,4 +22,32 @@ public void testValidateValidUrl() {
1822
Ip2GeoSettings.DatasourceEndpointValidator validator = new Ip2GeoSettings.DatasourceEndpointValidator();
1923
validator.validate("https://test.com");
2024
}
25+
26+
public void testDenyListDefaultValue() {
27+
List<String> privateNetworks = Arrays.asList(
28+
"127.0.0.0/8",
29+
"169.254.0.0/16",
30+
"10.0.0.0/8",
31+
"172.16.0.0/12",
32+
"192.168.0.0/16",
33+
"0.0.0.0/8",
34+
"100.64.0.0/10",
35+
"192.0.0.0/24",
36+
"192.0.2.0/24",
37+
"198.18.0.0/15",
38+
"192.88.99.0/24",
39+
"198.51.100.0/24",
40+
"203.0.113.0/24",
41+
"224.0.0.0/4",
42+
"240.0.0.0/4",
43+
"255.255.255.255/32",
44+
"::1/128",
45+
"fe80::/10",
46+
"fc00::/7",
47+
"::/128",
48+
"2001:db8::/32",
49+
"ff00::/8"
50+
);
51+
assertEquals(privateNetworks, Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY));
52+
}
2153
}

‎src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.io.IOException;
99
import java.time.Duration;
1010
import java.util.Arrays;
11+
import java.util.Collections;
1112
import java.util.HashMap;
1213
import java.util.List;
1314
import java.util.Locale;
@@ -17,6 +18,8 @@
1718

1819
import lombok.SneakyThrows;
1920

21+
import org.junit.AfterClass;
22+
import org.junit.BeforeClass;
2023
import org.opensearch.client.Response;
2124
import org.opensearch.client.ResponseException;
2225
import org.opensearch.common.Randomness;
@@ -25,6 +28,7 @@
2528
import org.opensearch.geospatial.GeospatialTestHelper;
2629
import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer;
2730
import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest;
31+
import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
2832

2933
public class Ip2GeoProcessorIT extends GeospatialRestTestCase {
3034
// Use this value in resource name to avoid name conflict among tests
@@ -34,9 +38,21 @@ public class Ip2GeoProcessorIT extends GeospatialRestTestCase {
3438
private static final String IP = "ip";
3539
private static final String SOURCE = "_source";
3640

41+
@BeforeClass
42+
public static void start() {
43+
Ip2GeoDataServer.start();
44+
}
45+
46+
@AfterClass
47+
public static void stop() {
48+
Ip2GeoDataServer.stop();
49+
}
50+
3751
@SneakyThrows
3852
public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() {
39-
Ip2GeoDataServer.start();
53+
// Reset deny list to allow private network access during test
54+
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));
55+
4056
boolean isDatasourceCreated = false;
4157
boolean isProcessorCreated = false;
4258
String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
@@ -109,13 +125,26 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() {
109125
} catch (Exception e) {
110126
exception = e;
111127
}
112-
Ip2GeoDataServer.stop();
113128
if (exception != null) {
114129
throw exception;
115130
}
116131
}
117132
}
118133

134+
@SneakyThrows
135+
public void testCreateIp2GeoProcessor_whenPrivateAddress_thenBlocked() {
136+
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
137+
Map<String, Object> datasourceProperties = Map.of(
138+
PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(),
139+
"http://127.0.0.1:9200/city/manifest_local.json"
140+
);
141+
142+
// Create datasource and wait for it to be available
143+
ResponseException exception = expectThrows(ResponseException.class, () -> createDatasource(datasourceName, datasourceProperties));
144+
assertEquals(400, exception.getResponse().getStatusLine().getStatusCode());
145+
assertTrue(exception.getMessage().contains("blocked by deny list"));
146+
}
147+
119148
private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map<String, Object> properties) throws IOException {
120149
String field = GeospatialTestHelper.randomLowerCaseString();
121150
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();

0 commit comments

Comments
 (0)