Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

Commit 3626fd1

Browse files
szhalanking520
szha
authored and
lanking520
committed
scripts for building libmxnet binary and wheel (#13648)
* add script for making all dependencies * tools for building pip package * build scripts for lib and wheel
1 parent a024a90 commit 3626fd1

File tree

12 files changed

+399
-252
lines changed

12 files changed

+399
-252
lines changed

‎tools/build/build_lib.sh‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the libraries of mxnet.
21+
make_config=config/pip_${PLATFORM}_${VARIANT}.mk
22+
if [[ ! -f $make_config ]]; then
23+
>&2 echo "Couldn't find make config $make_config for the current settings."
24+
exit 1
25+
fi
26+
27+
git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build
28+
29+
>&2 echo "Now building mxnet modules..."
30+
cp $make_config mxnet-build/config.mk
31+
32+
cd mxnet-build
33+
34+
make DEPS_PATH=$DEPS_PATH DMLCCORE
35+
make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a
36+
make DEPS_PATH=$DEPS_PATH PSLITE
37+
38+
if [[ $VARIANT == *mkl ]]; then
39+
MKLDNN_LICENSE='license.txt'
40+
if [[ $PLATFORM == 'linux' ]]; then
41+
IOMP_LIBFILE='libiomp5.so'
42+
MKLML_LIBFILE='libmklml_intel.so'
43+
MKLDNN_LIBFILE='libmkldnn.so.0'
44+
else
45+
IOMP_LIBFILE='libiomp5.dylib'
46+
MKLML_LIBFILE='libmklml.dylib'
47+
MKLDNN_LIBFILE='libmkldnn.0.dylib'
48+
fi
49+
make DEPS_PATH=$DEPS_PATH mkldnn
50+
cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE
51+
fi
52+
53+
if [[ $VARIANT == *mkl ]]; then
54+
>&2 echo "Copying MKL license."
55+
rm lib/libmkldnn.{so,dylib}
56+
rm lib/libmkldnn.0.*.dylib
57+
rm lib/libmkldnn.so.0.*
58+
fi
59+
60+
>&2 echo "Now building mxnet..."
61+
make DEPS_PATH=$DEPS_PATH || exit 1;
62+
63+
if [[ $PLATFORM == 'linux' ]]; then
64+
cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3
65+
cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0
66+
fi
67+
68+
# Print the linked objects on libmxnet.so
69+
>&2 echo "Checking linked objects on libmxnet.so..."
70+
if [[ ! -z $(command -v readelf) ]]; then
71+
readelf -d lib/libmxnet.so
72+
strip --strip-unneeded lib/libmxnet.so
73+
elif [[ ! -z $(command -v otool) ]]; then
74+
otool -L lib/libmxnet.so
75+
strip -u -r -x lib/libmxnet.so
76+
else
77+
>&2 echo "Not available"
78+
fi
79+
80+
cd ../

tools/pip_package/MANIFEST.in renamed to tools/build/build_wheel.sh

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env bash
2+
13
# Licensed to the Apache Software Foundation (ASF) under one
24
# or more contributor license agreements. See the NOTICE file
35
# distributed with this work for additional information
@@ -15,6 +17,15 @@
1517
# specific language governing permissions and limitations
1618
# under the License.
1719

18-
include README
19-
recursive-include * *.py
20-
recursive-include * *.so
20+
# This script builds the wheel for binary distribution and performs sanity check.
21+
22+
cd mxnet-build
23+
echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH
24+
cd -
25+
26+
# Make wheel for testing
27+
python setup.py bdist_wheel
28+
29+
wheel_name=$(ls -t dist | head -n 1)
30+
pip install -U --user --force-reinstall dist/$wheel_name
31+
python sanity_test.py

‎tools/dependencies/make_shared_dependencies.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This is a convenience script for calling the build scripts of all dependency libraries.
21+
# Environment variables should be set beforehand.
22+
23+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
24+
25+
26+
if [[ ! $PLATFORM == 'darwin' ]]; then
27+
source $DIR/openblas.sh
28+
fi
29+
source $DIR/libz.sh
30+
source $DIR/libturbojpeg.sh
31+
source $DIR/libpng.sh
32+
source $DIR/libtiff.sh
33+
source $DIR/openssl.sh
34+
source $DIR/curl.sh
35+
source $DIR/eigen.sh
36+
source $DIR/opencv.sh
37+
source $DIR/protobuf.sh
38+
source $DIR/cityhash.sh
39+
source $DIR/zmq.sh
40+
source $DIR/lz4.sh

‎tools/dependencies/opencv.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
# This script builds the static library of opencv that can be used as dependency of mxnet.
2121
# It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH.
2222
OPENCV_VERSION=3.4.2
23+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
2324
if [[ $PLATFORM == 'linux' ]]; then
2425
OPENCV_LAPACK_OPTIONS=" \
2526
-D OpenBLAS_HOME=$DEPS_PATH \
@@ -181,7 +182,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc
181182
-D CMAKE_BUILD_TYPE=RELEASE \
182183
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
183184
if [[ $PLATFORM == 'linux' ]]; then
184-
cp $DEPS_PATH/../patch/opencv_lapack.h ./
185+
cp $DIR/patch/opencv_lapack.h ./
185186
fi
186187
make
187188
make install

‎tools/dependencies/patch/opencv_lapack.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
extern "C" {
21+
#include "cblas.h"
22+
#include "lapacke.h"
23+
}

‎tools/dependencies/protobuf.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,5 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then
3939
make install
4040
cd -
4141
fi
42+
43+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib

‎tools/pip/MANIFEST.in‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
include README
2+
include mxnet/COMMIT_HASH
3+
recursive-include mxnet/tools *
4+
recursive-include mxnet *.py
5+
recursive-include mxnet *.so
6+
recursive-include mxnet *.so.*
7+
recursive-include mxnet *.dylib
8+
recursive-include mxnet *_LICENSE
9+
recursive-include mxnet *.h
10+
recursive-include mxnet *.cuh
11+
recursive-include dmlc_tracker *.py

‎tools/pip/sanity_test.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# coding: utf-8
19+
"""Sanity test."""
20+
from __future__ import print_function
21+
import sys
22+
from base64 import b64decode
23+
24+
try:
25+
import mxnet as mx
26+
mx.img.imdecode(b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')).asnumpy()
27+
print('Test succeeded')
28+
except:
29+
import traceback
30+
print('Test failed')
31+
traceback.print_exc()
32+
sys.exit(1)

0 commit comments

Comments
 (0)