Create Docker wheel builder

Dockerfile

@@ -1,44 +1,43 @@
-ARG BASE=11.5.0-devel-ubuntu20.04
+ARG BASE=12.2.0-devel-ubuntu22.04
 FROM nvidia/cuda:${BASE}
 
-LABEL maintainer="J.C.Knight@sussex.ac.uk"
-LABEL version="4.8.0"
-LABEL org.opencontainers.image.documentation="https://genn-team.github.io/"
-LABEL org.opencontainers.image.source="https://github.com/genn-team/genn"
-LABEL org.opencontainers.image.title="GeNN Docker image"
+ARG GENN_VER
+LABEL maintainer="J.C.Knight@sussex.ac.uk" \
+    version=${GENN_VER} \
+    org.opencontainers.image.documentation="https://genn-team.github.io/" \
+    org.opencontainers.image.source="https://github.com/genn-team/genn" \
+    org.opencontainers.image.title="GeNN Docker image"
 
-# Update APT database and upgrade any outdated packages
+# Update APT database and upgrade any outdated packages and install Python, pip and swig
 RUN apt-get update && \
-    apt-get upgrade -y
+    apt-get upgrade -y && \
+    apt-get install -yq --no-install-recommends python3-dev python3-pip swig gosu nano
 
-# Install Python, pip and swig
-RUN apt-get install -yq --no-install-recommends python3-dev python3-pip swig gosu nano
+# Set environment variables
+ENV CUDA_PATH=/usr/local/cuda \
+    GENN_PATH=/opt/genn
 
-# Set CUDA environment variable
-ENV CUDA_PATH=/usr/local/cuda
+# Set python3 to be the default version of python
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 
-ENV GENN_PATH=/opt/genn
-
-# Upgrade pip itself
-RUN pip install --upgrade pip
-
-# Install numpy and jupyter
-RUN pip install numpy jupyter matplotlib
+# Upgrade pip itself and install numpy and jupyter
+RUN python -m pip install --upgrade pip && \
+    python -m pip install numpy jupyter matplotlib
 
 # Copy GeNN into /opt
-COPY  . ${GENN_PATH}
+COPY . ${GENN_PATH}
 
 # Use this as working directory
 WORKDIR ${GENN_PATH}
 
 # Install GeNN and PyGeNN
 RUN make install -j `lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l`
 RUN make DYNAMIC=1 LIBRARY_DIRECTORY=${GENN_PATH}/pygenn/genn_wrapper/ -j `lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l`
-RUN python3 setup.py develop
-
-# Default command will be to launch bash
-CMD ["/bin/bash"]
+RUN python setup.py develop
 
 # Start entrypoint
 # **NOTE** in 'exec' mode shell arguments aren't expanded so can't use environment variables
 ENTRYPOINT ["/opt/genn/bin/genn-docker-entrypoint.sh"]
+
+# Default command will be to launch bash
+CMD ["/bin/bash"]

Dockerfile.builder

@@ -0,0 +1,31 @@
+# syntax=docker/dockerfile:1
+
+ARG CUDA=12.2
+FROM sameli/manylinux2014_x86_64_cuda_${CUDA} AS build
+
+ARG GENN_VER
+LABEL maintainer="B.D.Evans@sussex.ac.uk" \
+    org.opencontainers.image.documentation="https://genn-team.github.io/" \
+    org.opencontainers.image.source="https://github.com/genn-team/genn" \
+    org.opencontainers.image.title="PyGeNN wheel builder" \
+    org.opencontainers.image.cuda=${CUDA} \
+    org.opencontainers.image.genn=${GENN_VER}
+
+# Set environment variables
+ENV CUDA_PATH=/usr/local/cuda \
+    GENN_PATH=/opt/genn
+
+# Copy GeNN into /opt
+COPY . ${GENN_PATH}
+
+# Use this as working directory
+WORKDIR ${GENN_PATH}
+
+RUN ./build-wheels.sh
+
+# Copy the wheel to a new image for extraction
+FROM scratch AS output
+# TODO: Find a workaround for broken variable expansion
+#ARG GENN_PATH
+#COPY --from=build ${GENN_PATH}/dist/*.whl /
+COPY --from=build /opt/genn/dist/*.whl /

Dockerfile.ubuntu_builder

@@ -0,0 +1,50 @@
+# syntax=docker/dockerfile:1
+
+ARG BASE=12.2.0-devel-ubuntu22.04
+FROM nvidia/cuda:${BASE} AS build
+
+ARG PY_VER=3.11
+ARG GENN_VER
+LABEL maintainer="B.D.Evans@sussex.ac.uk" \
+    org.opencontainers.image.documentation="https://genn-team.github.io/" \
+    org.opencontainers.image.source="https://github.com/genn-team/genn" \
+    org.opencontainers.image.title="PyGeNN wheel builder" \
+    org.opencontainers.image.cuda=${BASE} \
+    org.opencontainers.image.python=${PY_VER} \
+    org.opencontainers.image.genn=${GENN_VER}
+
+# Update APT database and upgrade any outdated packages and install Python and pip
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -yq --no-install-recommends python${PY_VER}-dev python3-pip
+
+# Set environment variables
+ENV CUDA_PATH=/usr/local/cuda \
+    GENN_PATH=/opt/genn
+
+# Set python3 to be the default version of python
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python${PY_VER} 1
+
+# Upgrade pip itself and install numpy and swig
+RUN python -m pip install --upgrade pip && \
+    python -m pip install numpy swig
+
+# Copy GeNN into /opt
+COPY . ${GENN_PATH}
+
+# Use this as working directory
+WORKDIR ${GENN_PATH}
+
+# Install GeNN and PyGeNN
+# RUN make install -j `lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l`
+RUN make DYNAMIC=1 LIBRARY_DIRECTORY=${GENN_PATH}/pygenn/genn_wrapper/ -j `lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l`
+# RUN python3 setup.py develop
+RUN python setup.py bdist_wheel
+RUN python setup.py bdist_wheel
+
+# Copy the wheel to a new image for extraction
+FROM scratch AS output
+# TODO: Find a workaround for broken variable expansion
+#ARG GENN_PATH
+#COPY --from=build ${GENN_PATH}/dist/*.whl /
+COPY --from=build /opt/genn/dist/*.whl /

Makefile

@@ -70,6 +70,41 @@ clean:
 	@rm -f $(LIBGENN)
 	@rm -f $(BACKEND_LIBS)
 
+GENN_VER := $(shell cat version.txt)
 .PHONY docker-build:
 docker-build:
-	@docker build -t genn:latest .
+	@docker build --build-arg GENN_VER=$(GENN_VER) -t genn:latest .
+
+BASE := 12.2.0-devel-ubuntu22.04
+PY_VER := 3.11
+.PHONY ubuntu_wheel:
+ubuntu_wheel:
+	@docker build -f Dockerfile.ubuntu_builder \
+		--build-arg BASE=$(BASE) \
+		--build-arg PY_VER=$(PY_VER) \
+		--build-arg GENN_VER=$(GENN_VER) \
+		--target=output --output type=local,dest=dist/ .
+
+CUDA := 12.2
+.PHONY wheels:
+wheels:
+	@docker build -f Dockerfile.builder \
+		--build-arg CUDA=$(CUDA) \
+		--build-arg GENN_VER=$(GENN_VER) \
+		--target=output --output type=local,dest=dist/$(CUDA)/ .
+
+# Build wheels for all supported CUDA versions: https://github.com/ameli/manylinux-cuda
+SUPPORTED_CUDA = 12.2 12.0 11.8 11.7 10.2
+.PHONY all_wheels:
+all_wheels:
+	for cuda in $(SUPPORTED_CUDA); do \
+		docker build -f Dockerfile.builder \
+			--build-arg CUDA=$$cuda \
+			--build-arg GENN_VER=$(GENN_VER) \
+			--target=output --output type=local,dest=dist/$$cuda/ .; \
+	done
+
+# TODO: Consider build with docker run instead of docker build
+# See: https://github.com/pypa/python-manylinux-demo
+# PLAT=manylinux2014_x86_64
+# docker run --rm -e PLAT=$PLAT -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/travis/build-wheels.sh

build-wheels.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e -u -x
+
+# Adapted from: https://github.com/pypa/python-manylinux-demo/blob/master/travis/build-wheels.sh
+
+PLAT=manylinux2014_x86_64
+
+function repair_wheel {
+    wheel="$1"
+    if ! auditwheel show "$wheel"; then
+        echo "Skipping non-platform wheel $wheel"
+    else
+        auditwheel repair "$wheel" --plat "$PLAT" -w /opt/genn/dist/
+    fi
+}
+
+
+# # Install a system package required by our library
+# yum install -y atlas-devel
+
+# make install -j `lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l`
+make DYNAMIC=1 LIBRARY_DIRECTORY=${GENN_PATH}/pygenn/genn_wrapper/ -j `lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l`
+
+# Compile wheels
+for PYBIN in /opt/python/*/bin; do
+    # Only build for the following versions of cPython (exclude pypy, EOL and beta versions)
+    # https://devguide.python.org/versions/
+    if [[ "$PYBIN" == *"cp38"* || "$PYBIN" == *"cp39"* || "$PYBIN" == *"cp310"* || "$PYBIN" == *"cp311"* ]]; then
+        # "${PYBIN}/pip" install -r /io/dev-requirements.txt
+        "${PYBIN}/pip" install numpy swig
+        # "${PYBIN}/pip" wheel /opt/genn/ --no-deps -w dist/
+        "${PYBIN}/python" setup.py bdist_wheel
+        "${PYBIN}/python" setup.py bdist_wheel
+    fi
+done
+
+# Bundle external shared libraries into the wheels
+for whl in dist/*.whl; do
+    repair_wheel "$whl"
+done
+
+# # Install packages and test
+# for PYBIN in /opt/python/*/bin/; do
+#     "${PYBIN}/pip" install python-manylinux-demo --no-index -f /opt/genn/dist
+#     (cd "$HOME"; "${PYBIN}/nosetests" pymanylinuxdemo)
+# done