Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

Commit 439f167

Browse files
szhalanking520
szha
authored and
lanking520
committed
Scripts for building dependency libraries of MXNet (#13282)
* openblas script * ps-lite dependencies * USE_S3 dependencies * image libraries * license
1 parent d1e7b81 commit 439f167

File tree

14 files changed

+673
-0
lines changed

14 files changed

+673
-0
lines changed

‎tools/dependencies/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Overview
2+
3+
This folder contains scripts for building the dependencies from source. The static libraries from
4+
the build artifacts can be used to create self-contained shared object for mxnet through static
5+
linking.
6+
7+
# Settings
8+
9+
The scripts use the following environment variables for setting behavior:
10+
11+
`DEPS_PATH`: the location in which the libraries are downloaded, built, and installed.
12+
`PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'.
13+
14+
It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm

‎tools/dependencies/cityhash.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of cityhash that can be used as dependency of mxnet.
21+
CITYHASH_VERSION=1.1.1
22+
if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
23+
# Download and build cityhash
24+
>&2 echo "Building cityhash..."
25+
git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION
26+
cd $DEPS_PATH/cityhash-$CITYHASH_VERSION
27+
git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db
28+
./configure -prefix=$DEPS_PATH --enable-sse4.2
29+
make CXXFLAGS="-g -O3 -msse4.2"
30+
make install
31+
cd -
32+
fi

‎tools/dependencies/curl.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of libcurl that can be used as dependency of mxnet.
21+
LIBCURL_VERSION=7.61.0
22+
if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
23+
# download and build libcurl
24+
>&2 echo "Building libcurl..."
25+
curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip
26+
unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH
27+
cd $DEPS_PATH/curl-$LIBCURL_VERSION
28+
if [[ $PLATFORM == 'linux' ]]; then
29+
CONFIG_FLAG=""
30+
elif [[ $PLATFORM == 'darwin' ]]; then
31+
CONFIG_FLAG="--with-darwinssl"
32+
fi
33+
./configure $CONFIG_FLAG \
34+
--with-zlib \
35+
--with-nghttps2 \
36+
--without-zsh-functions-dir \
37+
--without-librtmp \
38+
--without-libssh2 \
39+
--disable-debug \
40+
--disable-curldebug \
41+
--enable-symbol-hiding=yes \
42+
--enable-optimize=yes \
43+
--enable-shared=no \
44+
--enable-http=yes \
45+
--enable-ipv6=yes \
46+
--disable-ftp \
47+
--disable-ldap \
48+
--disable-ldaps \
49+
--disable-rtsp \
50+
--disable-proxy \
51+
--disable-dict \
52+
--disable-telnet \
53+
--disable-tftp \
54+
--disable-pop3 \
55+
--disable-imap \
56+
--disable-smb \
57+
--disable-smtp \
58+
--disable-gopher \
59+
--disable-manual \
60+
--prefix=$DEPS_PATH
61+
make
62+
make install
63+
cd -
64+
fi

‎tools/dependencies/eigen.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script imports the headers from eigen3 that can be used to in opencv.
21+
EIGEN_VERSION=3.3.4
22+
if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
23+
# download eigen
24+
>&2 echo "Loading eigen..."
25+
curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip
26+
unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH
27+
mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
28+
cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
29+
cmake \
30+
-D CMAKE_BUILD_TYPE=RELEASE \
31+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
32+
make install
33+
cd -
34+
fi

‎tools/dependencies/libpng.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of libpng that can be used as dependency of mxnet/opencv.
21+
PNG_VERSION=1.6.34
22+
if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
23+
# download and build libpng
24+
>&2 echo "Building libpng..."
25+
curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip
26+
unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH
27+
mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build
28+
cd $DEPS_PATH/libpng-$PNG_VERSION/build
29+
cmake \
30+
-D PNG_SHARED=OFF \
31+
-D PNG_STATIC=ON \
32+
-D CMAKE_BUILD_TYPE=RELEASE \
33+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
34+
-D CMAKE_C_FLAGS=-fPIC ..
35+
make
36+
make install
37+
mkdir -p $DEPS_PATH/include/libpng
38+
ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h
39+
cd -
40+
fi

‎tools/dependencies/libtiff.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of libtiff that can be used as dependency of mxnet/opencv.
21+
TIFF_VERSION="4-0-9"
22+
if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then
23+
# download and build libtiff
24+
>&2 echo "Building libtiff..."
25+
curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip
26+
unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH
27+
cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION
28+
./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH
29+
make
30+
make install
31+
cd -
32+
fi

‎tools/dependencies/libturbojpeg.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of libturbojpeg that can be used as dependency of
21+
# mxnet/opencv.
22+
TURBO_JPEG_VERSION=1.5.90
23+
if [[ $PLATFORM == 'darwin' ]]; then
24+
JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
25+
fi
26+
27+
if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
28+
# download and build libjpeg
29+
>&2 echo "Building libjpeg-turbo..."
30+
curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip
31+
unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
32+
mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
33+
cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
34+
cmake \
35+
-G"Unix Makefiles" \
36+
-D CMAKE_BUILD_TYPE=RELEASE \
37+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
38+
-D CMAKE_C_FLAGS=-fPIC \
39+
-D WITH_JAVA=FALSE \
40+
-D WITH_JPEG7=TRUE \
41+
-D WITH_JPEG8=TRUE \
42+
$JPEG_NASM_OPTION \
43+
-D ENABLE_SHARED=FALSE ..
44+
make
45+
make install
46+
cd -
47+
fi

‎tools/dependencies/libz.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of libz that can be used as dependency of mxnet.
21+
ZLIB_VERSION=1.2.6
22+
if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
23+
# Download and build zlib
24+
>&2 echo "Building zlib..."
25+
curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip
26+
unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH
27+
mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build
28+
cd $DEPS_PATH/zlib-$ZLIB_VERSION/build
29+
cmake \
30+
-D CMAKE_BUILD_TYPE=RELEASE \
31+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
32+
-D BUILD_SHARED_LIBS=OFF ..
33+
make
34+
make install
35+
cd -
36+
fi

‎tools/dependencies/lz4.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of lz4 that can be used as dependency of mxnet.
21+
LZ4_VERSION=r130
22+
if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
23+
# Download and build lz4
24+
>&2 echo "Building lz4..."
25+
curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip
26+
unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH
27+
cd $DEPS_PATH/lz4-$LZ4_VERSION
28+
make
29+
make PREFIX=$DEPS_PATH install
30+
cd -
31+
fi

‎tools/dependencies/openblas.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# This script builds the static library of openblas that can be used as dependency of mxnet.
21+
OPENBLAS_VERSION=0.3.3
22+
if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
23+
# download and build openblas
24+
>&2 echo "Building openblas..."
25+
26+
curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip
27+
unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH
28+
cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION
29+
30+
make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1
31+
make PREFIX=$DEPS_PATH install
32+
cd -
33+
ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a
34+
ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a
35+
fi

0 commit comments

Comments
 (0)