Skip to content

Commit 5a06d9e

Browse files
robbie228robbie228
authored
Merge pull request #5741 from FederatedAI/feature-2.2.0-docker_compose
Feature 2.2.0 docker compose
2 parents f418070 + 60f939a commit 5a06d9e

22 files changed

+2340
-0
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ FATE can be deployed on a single node or on multiple nodes. Choose the deploymen
3333
### Cluster deployment
3434
Deploying FATE to multiple nodes to achieve scalability, reliability and manageability.
3535
- [Cluster deployment by CLI](./deploy/cluster-deploy): Using CLI to deploy a FATE cluster.
36+
- [Docker-Compose deployment](./deploy/docker-compose): Using docker-compose to deploy FATE.
3637

3738
### Quick Start
3839
- [Training Demo with Only FATE Installed From Pypi](doc/2.0/fate/ml)

‎deploy/docker-compose/README_zh.md‎

Lines changed: 411 additions & 0 deletions
Large diffs are not rendered by default.

‎deploy/docker-compose/docker-deploy/docker_deploy.sh‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
#!/bin/bash
2+
3+
# Copyright 2019-2022 VMware, Inc.
4+
# Copyright 2019 The FATE Authors. All Rights Reserved.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# you may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -e
17+
BASEDIR=$(dirname "$0")
18+
cd $BASEDIR
19+
WORKINGDIR=$(pwd)
20+
21+
# fetch fate-python image
22+
source ${WORKINGDIR}/.env
23+
source ${WORKINGDIR}/parties.conf
24+
cd ${WORKINGDIR}
25+
26+
Deploy() {
27+
if [ "$1" = "" ]; then
28+
echo "No party id was provided, please check your arguments "
29+
exit 1
30+
fi
31+
32+
while [ "$1" != "" ]; do
33+
case $1 in
34+
splitting_proxy)
35+
shift
36+
DeployPartyInternal $@
37+
break
38+
;;
39+
all)
40+
for party in ${party_list[*]}; do
41+
if [ "$2" != "" ]; then
42+
case $2 in
43+
--training)
44+
DeployPartyInternal $party
45+
if [ "${exchangeip}" != "" ]; then
46+
DeployPartyInternal exchange
47+
fi
48+
;;
49+
esac
50+
else
51+
DeployPartyInternal $party
52+
if [ "${exchangeip}" != "" ]; then
53+
DeployPartyInternal exchange
54+
fi
55+
fi
56+
done
57+
break
58+
;;
59+
*)
60+
if [ "$2" != "" ]; then
61+
case $2 in
62+
--training)
63+
DeployPartyInternal $1
64+
break
65+
;;
66+
esac
67+
else
68+
DeployPartyInternal $1
69+
fi
70+
;;
71+
esac
72+
shift
73+
74+
done
75+
}
76+
77+
Delete() {
78+
if [ "$1" = "" ]; then
79+
echo "No party id was provided, please check your arguments "
80+
exit 1
81+
fi
82+
83+
while [ "$1" != "" ]; do
84+
case $1 in
85+
all)
86+
for party in ${party_list[*]}; do
87+
if [ "$2" != "" ]; then
88+
DeleteCluster $party $2
89+
else
90+
DeleteCluster $party
91+
fi
92+
done
93+
if [ "${exchangeip}" != "" ]; then
94+
DeleteCluster exchange
95+
fi
96+
break
97+
;;
98+
*)
99+
DeleteCluster $@
100+
break
101+
;;
102+
esac
103+
done
104+
}
105+
106+
DeployPartyInternal() {
107+
target_party_id=$1
108+
# should not use localhost at any case
109+
target_party_ip="127.0.0.1"
110+
111+
# check configuration files
112+
if [ ! -d ${WORKINGDIR}/outputs ]; then
113+
echo "Unable to find outputs dir, please generate config files first."
114+
return 1
115+
fi
116+
if [ ! -f ${WORKINGDIR}/outputs/confs-${target_party_id}.tar ]; then
117+
echo "Unable to find deployment file of training for party $target_party_id, please generate it first."
118+
return 0
119+
fi
120+
# extract the ip address of the target party
121+
if [ "$target_party_id" = "exchange" ]; then
122+
target_party_ip=${exchangeip}
123+
elif [ "$2" != "" ]; then
124+
target_party_ip="$2"
125+
else
126+
for ((i = 0; i < ${#party_list[*]}; i++)); do
127+
if [ "${party_list[$i]}" = "$target_party_id" ]; then
128+
target_party_ip=${party_ip_list[$i]}
129+
fi
130+
done
131+
fi
132+
# verify the target_party_ip
133+
if [ "$target_party_ip" = "127.0.0.1" ]; then
134+
echo "Unable to find Party: $target_party_id, please check you input."
135+
return 1
136+
fi
137+
if [ "$3" != "" ]; then
138+
user=$3
139+
fi
140+
141+
handleLocally confs
142+
if [ "$local_flag" == "true" ]; then
143+
return 0
144+
fi
145+
146+
scp -P ${SSH_PORT} ${WORKINGDIR}/outputs/confs-$target_party_id.tar $user@$target_party_ip:~/
147+
#rm -f ${WORKINGDIR}/outputs/confs-$target_party_id.tar
148+
echo "$target_party_ip training cluster copy is ok!"
149+
ssh -p ${SSH_PORT} -tt $user@$target_party_ip << eeooff
150+
mkdir -p $dir
151+
rm -f $dir/confs-$target_party_id.tar
152+
mv ~/confs-$target_party_id.tar $dir
153+
cd $dir
154+
tar -xzf confs-$target_party_id.tar
155+
cd confs-$target_party_id
156+
docker-compose down
157+
docker volume rm -f confs-${target_party_id}_shared_dir_examples
158+
docker volume rm -f confs-${target_party_id}_shared_dir_fate
159+
docker volume rm -f confs-${target_party_id}_sdownload_dir
160+
docker volume rm -f confs-${target_party_id}_fate_flow_logs
161+
162+
docker-compose up -d
163+
cd ../
164+
rm -f confs-${target_party_id}.tar
165+
exit
166+
eeooff
167+
echo "party ${target_party_id} deploy is ok!"
168+
}
169+
170+
DeleteCluster() {
171+
target_party_id=$1
172+
cluster_type=$2
173+
target_party_serving_ip="127.0.0.1"
174+
target_party_ip="127.0.0.1"
175+
176+
# extract the ip address of the target party
177+
if [ "$target_party_id" == "exchange" ]; then
178+
target_party_ip=${exchangeip}
179+
else
180+
for ((i = 0; i < ${#party_list[*]}; i++)); do
181+
if [ "${party_list[$i]}" = "$target_party_id" ]; then
182+
target_party_ip=${party_ip_list[$i]}
183+
fi
184+
done
185+
fi
186+
187+
# echo "target_party_ip: $target_party_ip"
188+
189+
for ((i = 0; i < ${#party_list[*]}; i++)); do
190+
if [ "${party_list[$i]}" = "$target_party_id" ]; then
191+
target_party_serving_ip=${serving_ip_list[$i]}
192+
fi
193+
done
194+
195+
# echo "target_party_ip: $target_party_ip"
196+
# echo "cluster_type: $cluster_type"
197+
198+
# delete training cluster
199+
if [ "$cluster_type" == "--training" ]; then
200+
ssh -p ${SSH_PORT} -tt $user@$target_party_ip <<eeooff
201+
cd $dir/confs-$target_party_id
202+
docker-compose down
203+
exit
204+
eeooff
205+
echo "party $target_party_id training cluster is deleted!"
206+
# delete training cluster and serving cluster
207+
else
208+
# if party is exchange then delete exchange cluster
209+
if [ "$target_party_id" == "exchange" ]; then
210+
ssh -p ${SSH_PORT} -tt $user@$target_party_ip <<eeooff
211+
cd $dir/confs-$target_party_id
212+
docker-compose down
213+
exit
214+
eeooff
215+
else
216+
if [ "$target_party_ip" != "" ]; then
217+
ssh -p ${SSH_PORT} -tt $user@$target_party_ip <<eeooff
218+
cd $dir/confs-$target_party_id
219+
docker-compose down
220+
exit
221+
eeooff
222+
fi
223+
echo "party $target_party_id training cluster is deleted!"
224+
fi
225+
fi
226+
}
227+
228+
ShowUsage() {
229+
echo "Usage: "
230+
echo "Deploy all parties or specified partie(s): bash docker_deploy.sh partyid1[partyid2...] | all"
231+
}
232+
233+
handleLocally() {
234+
type=$1
235+
for ip in $(hostname -I); do
236+
if [ "$target_party_ip" == "$ip" ]; then
237+
mkdir -p $dir
238+
tar -xf ${WORKINGDIR}/outputs/${type}-${target_party_id}.tar -C $dir
239+
cd ${dir}/${type}-${target_party_id}
240+
docker-compose down
241+
docker-compose up -d
242+
local_flag="true"
243+
return 0
244+
fi
245+
done
246+
local_flag="false"
247+
}
248+
249+
main() {
250+
251+
if [ "$1" = "" ] || [ "$1" = "--help" ]; then
252+
ShowUsage
253+
exit 1
254+
elif [ "$1" = "--delete" ] || [ "$1" = "--del" ]; then
255+
shift
256+
Delete $@
257+
else
258+
Deploy "$@"
259+
fi
260+
261+
for ((i = 0; i < ${#party_list[*]}; i++)); do
262+
if [ $party_list[$i] != "exchange" ]; then
263+
echo "Use ${party_ip_list[$i]}:8080 to access fateboard of party: ${party_list[$i]}"
264+
fi
265+
if [[ "$computing" == "spark"* ]]; then
266+
echo " Use ${party_ip_list[$i]}:8888 to access Spark of party: ${party_list[$i]}"
267+
fi
268+
done
269+
270+
exit 0
271+
}
272+
273+
main $@

0 commit comments

Comments
 (0)