Skip to content

Commit 8873210

Browse files
BrendanSchellBrendanSchell
authored
Merge pull request #9304 from OpenMined/bschell/add-l0-worker-pool-testing
Add L0 worker pool testing notebook
2 parents 8604131 + bca09d9 commit 8873210

File tree

6 files changed

+387
-13
lines changed

6 files changed

+387
-13
lines changed

notebooks/scenarios/bigquery/sync/01-setup-high-low-datasites.ipynb renamed to notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
"source": [
3232
"# stdlib\n",
3333
"environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n",
34-
"high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n",
35-
"low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n",
34+
"high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n",
35+
"low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n",
3636
"print(environment, high_port, low_port)"
3737
]
3838
},
@@ -327,7 +327,7 @@
327327
"name": "python",
328328
"nbconvert_exporter": "python",
329329
"pygments_lexer": "ipython3",
330-
"version": "3.12.3"
330+
"version": "3.12.4"
331331
}
332332
},
333333
"nbformat": 4,

‎notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb‎

Lines changed: 374 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,374 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "0",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"# stdlib\n",
11+
"import os\n",
12+
"\n",
13+
"# Testing works over 4 possibilities\n",
14+
"# 1. (python/in-memory workers and using tox commands)\n",
15+
"# 2. (python/in-memory workers and manually running notebooks)\n",
16+
"# 3. (using k8s and using tox commands)\n",
17+
"# 4. (using k8s and manually running notebooks)\n",
18+
"# Uncomment the lines below if in the 4th possibility\n",
19+
"\n",
20+
"# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n",
21+
"# os.environ[\"DEV_MODE\"] = \"True\"\n",
22+
"# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n",
23+
"# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n",
24+
"# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\""
25+
]
26+
},
27+
{
28+
"cell_type": "code",
29+
"execution_count": null,
30+
"id": "1",
31+
"metadata": {},
32+
"outputs": [],
33+
"source": [
34+
"# stdlib\n",
35+
"environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n",
36+
"high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n",
37+
"low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n",
38+
"print(environment, high_port, low_port)"
39+
]
40+
},
41+
{
42+
"cell_type": "code",
43+
"execution_count": null,
44+
"id": "2",
45+
"metadata": {},
46+
"outputs": [],
47+
"source": [
48+
"# stdlib\n",
49+
"import os\n",
50+
"\n",
51+
"# syft absolute\n",
52+
"import syft as sy\n",
53+
"from syft.util.test_helpers.email_helpers import Timeout\n",
54+
"from syft.util.test_helpers.email_helpers import get_email_server"
55+
]
56+
},
57+
{
58+
"cell_type": "code",
59+
"execution_count": null,
60+
"id": "3",
61+
"metadata": {},
62+
"outputs": [],
63+
"source": [
64+
"num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n",
65+
"\n",
66+
"# ROOT_EMAIL = \"admin@bigquery.org\"\n",
67+
"# ROOT_PASSWORD = \"bqpw\"\n",
68+
"environment"
69+
]
70+
},
71+
{
72+
"cell_type": "markdown",
73+
"id": "4",
74+
"metadata": {},
75+
"source": [
76+
"### Launch server & login"
77+
]
78+
},
79+
{
80+
"cell_type": "code",
81+
"execution_count": null,
82+
"id": "5",
83+
"metadata": {},
84+
"outputs": [],
85+
"source": [
86+
"server_low = sy.orchestra.launch(\n",
87+
" name=\"bigquery-low\",\n",
88+
" server_side_type=\"low\",\n",
89+
" dev_mode=True,\n",
90+
" n_consumers=1,\n",
91+
" create_producer=True,\n",
92+
" port=low_port,\n",
93+
")"
94+
]
95+
},
96+
{
97+
"cell_type": "code",
98+
"execution_count": null,
99+
"id": "6",
100+
"metadata": {},
101+
"outputs": [],
102+
"source": [
103+
"email_server, smtp_server = get_email_server(reset=True)"
104+
]
105+
},
106+
{
107+
"cell_type": "code",
108+
"execution_count": null,
109+
"id": "7",
110+
"metadata": {},
111+
"outputs": [],
112+
"source": [
113+
"low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")"
114+
]
115+
},
116+
{
117+
"cell_type": "code",
118+
"execution_count": null,
119+
"id": "8",
120+
"metadata": {},
121+
"outputs": [],
122+
"source": [
123+
"assert len(low_client.worker_pools.get_all()) == 2"
124+
]
125+
},
126+
{
127+
"cell_type": "code",
128+
"execution_count": null,
129+
"id": "9",
130+
"metadata": {},
131+
"outputs": [],
132+
"source": [
133+
"default_worker_pool = low_client.worker_pools.get_by_name(\"default-pool\")\n",
134+
"default_worker_pool"
135+
]
136+
},
137+
{
138+
"cell_type": "markdown",
139+
"id": "10",
140+
"metadata": {},
141+
"source": [
142+
"### Scale Worker pool"
143+
]
144+
},
145+
{
146+
"cell_type": "markdown",
147+
"id": "11",
148+
"metadata": {},
149+
"source": [
150+
"##### Scale up"
151+
]
152+
},
153+
{
154+
"cell_type": "code",
155+
"execution_count": null,
156+
"id": "12",
157+
"metadata": {},
158+
"outputs": [],
159+
"source": [
160+
"# Scale to 1\n",
161+
"if environment == \"remote\":\n",
162+
" low_client.api.worker_pool.scale(\n",
163+
" number=num_workers, pool_name=default_worker_pool.name\n",
164+
" )"
165+
]
166+
},
167+
{
168+
"cell_type": "code",
169+
"execution_count": null,
170+
"id": "13",
171+
"metadata": {},
172+
"outputs": [],
173+
"source": [
174+
"low_client.api.services.worker_pool[0]"
175+
]
176+
},
177+
{
178+
"cell_type": "code",
179+
"execution_count": null,
180+
"id": "14",
181+
"metadata": {},
182+
"outputs": [],
183+
"source": [
184+
"# Scale up workers\n",
185+
"if environment == \"remote\":\n",
186+
" scale_up_result = low_client.api.worker_pool.scale(\n",
187+
" number=5, pool_name=default_worker_pool.name\n",
188+
" )\n",
189+
" if environment == \"remote\":\n",
190+
" assert scale_up_result, scale_up_result\n",
191+
"\n",
192+
" assert (\n",
193+
" low_client.api.services.worker_pool[default_worker_pool.name].max_count == 5\n",
194+
" )"
195+
]
196+
},
197+
{
198+
"cell_type": "markdown",
199+
"id": "15",
200+
"metadata": {},
201+
"source": [
202+
"##### Scale down"
203+
]
204+
},
205+
{
206+
"cell_type": "code",
207+
"execution_count": null,
208+
"id": "16",
209+
"metadata": {},
210+
"outputs": [],
211+
"source": [
212+
"# Scale down workers, this gracefully shutdowns the consumers\n",
213+
"if environment == \"remote\":\n",
214+
" scale_down_result = low_client.api.worker_pool.scale(\n",
215+
" number=num_workers, pool_name=default_worker_pool.name\n",
216+
" )\n",
217+
" assert scale_down_result, scale_down_result"
218+
]
219+
},
220+
{
221+
"cell_type": "code",
222+
"execution_count": null,
223+
"id": "17",
224+
"metadata": {},
225+
"outputs": [],
226+
"source": [
227+
"if environment == \"remote\":\n",
228+
"\n",
229+
" def has_worker_scaled_down():\n",
230+
" return (\n",
231+
" low_client.api.worker_pool[default_worker_pool.name].max_count\n",
232+
" == num_workers\n",
233+
" )\n",
234+
"\n",
235+
" worker_scale_timeout = Timeout(timeout_duration=20)\n",
236+
" worker_scale_timeout.run_with_timeout(has_worker_scaled_down)"
237+
]
238+
},
239+
{
240+
"cell_type": "code",
241+
"execution_count": null,
242+
"id": "18",
243+
"metadata": {},
244+
"outputs": [],
245+
"source": [
246+
"if environment == \"remote\":\n",
247+
" assert (\n",
248+
" low_client.api.services.worker_pool[default_worker_pool.name].max_count\n",
249+
" == num_workers\n",
250+
" )"
251+
]
252+
},
253+
{
254+
"cell_type": "markdown",
255+
"id": "19",
256+
"metadata": {},
257+
"source": [
258+
"#### Delete Worker Pool"
259+
]
260+
},
261+
{
262+
"cell_type": "code",
263+
"execution_count": null,
264+
"id": "20",
265+
"metadata": {},
266+
"outputs": [],
267+
"source": [
268+
"pool_delete_result = low_client.api.services.worker_pool.delete(\n",
269+
" pool_name=default_worker_pool.name\n",
270+
")\n",
271+
"pool_delete_result"
272+
]
273+
},
274+
{
275+
"cell_type": "code",
276+
"execution_count": null,
277+
"id": "21",
278+
"metadata": {},
279+
"outputs": [],
280+
"source": [
281+
"with sy.raises(KeyError):\n",
282+
" _ = low_client.api.services.worker_pool[default_worker_pool.name]"
283+
]
284+
},
285+
{
286+
"cell_type": "markdown",
287+
"id": "22",
288+
"metadata": {},
289+
"source": [
290+
"#### Re-launch the default worker pool"
291+
]
292+
},
293+
{
294+
"cell_type": "code",
295+
"execution_count": null,
296+
"id": "23",
297+
"metadata": {},
298+
"outputs": [],
299+
"source": [
300+
"default_worker_image = default_worker_pool.image"
301+
]
302+
},
303+
{
304+
"cell_type": "code",
305+
"execution_count": null,
306+
"id": "24",
307+
"metadata": {},
308+
"outputs": [],
309+
"source": [
310+
"launch_result = low_client.api.services.worker_pool.launch(\n",
311+
" pool_name=default_worker_pool.name,\n",
312+
" image_uid=default_worker_image.id,\n",
313+
" num_workers=num_workers,\n",
314+
")"
315+
]
316+
},
317+
{
318+
"cell_type": "code",
319+
"execution_count": null,
320+
"id": "25",
321+
"metadata": {},
322+
"outputs": [],
323+
"source": [
324+
"assert low_client.api.services.worker_pool[default_worker_pool.name]\n",
325+
"assert (\n",
326+
" low_client.api.services.worker_pool[default_worker_pool.name].max_count\n",
327+
" == num_workers\n",
328+
")"
329+
]
330+
},
331+
{
332+
"cell_type": "code",
333+
"execution_count": null,
334+
"id": "26",
335+
"metadata": {},
336+
"outputs": [],
337+
"source": [
338+
"smtp_server.stop()"
339+
]
340+
},
341+
{
342+
"cell_type": "code",
343+
"execution_count": null,
344+
"id": "27",
345+
"metadata": {},
346+
"outputs": [],
347+
"source": [
348+
"if environment != \"remote\":\n",
349+
" server_low.land()"
350+
]
351+
}
352+
],
353+
"metadata": {
354+
"kernelspec": {
355+
"display_name": "syft",
356+
"language": "python",
357+
"name": "python3"
358+
},
359+
"language_info": {
360+
"codemirror_mode": {
361+
"name": "ipython",
362+
"version": 3
363+
},
364+
"file_extension": ".py",
365+
"mimetype": "text/x-python",
366+
"name": "python",
367+
"nbconvert_exporter": "python",
368+
"pygments_lexer": "ipython3",
369+
"version": "3.12.4"
370+
}
371+
},
372+
"nbformat": 4,
373+
"nbformat_minor": 5
374+
}

0 commit comments

Comments
 (0)