diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e2aaff1d..d36f68dc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,6 +45,16 @@ jobs: - name: Format check for C run: git diff --exit-code + - uses: actions/upload-artifact@v3 + name: Upload artifacts + if: ${{ matrix.java-version == 8 }} + with: + name: docker-built-shared-objects + path: | + src/main/resources/*/*.so + src/main/resources/*/*.dll + if-no-files-found: error + tests-new-dockcross: name: Dockcross ${{ matrix.dockcross-tag }} Java ${{ matrix.java-version }} ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -54,7 +64,7 @@ jobs: os: [ubuntu-latest] java-distribution: [adopt] java-version: [17] - dockcross-tag: ["20220906-e88a3ce"] + dockcross-tag: ["20220906-e88a3ce", "20230116-670f7f7"] steps: - uses: actions/checkout@v2.1.1 @@ -109,6 +119,14 @@ jobs: - name: Tests run: mvn -B -V clean test site + - uses: actions/upload-artifact@v3 + name: Upload Mac OS Artifacts + if: ${{ matrix.os == 'macos-latest' && matrix.java-version == 8 }} + with: + name: macos-built-shared-objects + path: src/main/resources/*/*.dylib + if-no-files-found: error + tests-coverage: name: Java (Coverage) ${{ matrix.java-version }} ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -143,3 +161,52 @@ jobs: env: CI_NAME: github COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} + + tests-use-built-artifacts: + name: Java (Built Artifacts) ${{ matrix.java-version }} ${{ matrix.os }} + needs: + - tests + - tests-no-docker + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest] + java-distribution: [adopt] + java-version: [8] + + steps: + - uses: actions/checkout@v2.1.1 + with: + submodules: recursive + + - uses: actions/setup-java@v2 + with: + distribution: "${{ matrix.java-distribution }}" + java-version: "${{ matrix.java-version }}" + + - uses: actions/cache@v2 + id: maven-cache + with: + path: ~/.m2/ + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Download Docker binaries + uses: actions/download-artifact@v3 + with: + name: docker-built-shared-objects + path: src/main/resources/ + + - name: Download Mac binaries + uses: actions/download-artifact@v3 + with: + name: macos-built-shared-objects + path: src/main/resources/ + + - name: Download and test + run: | + mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.by_run=true + env: + GH_TOKEN: ${{ github.token }} diff --git a/docs/releasing.md b/docs/releasing.md index 3f7b7492..c55e5875 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -8,13 +8,18 @@ Release builds should only be run on Mac OSX with Docker. This is needed so that 1. Ensure you are on branch `master` and that the library is building correctly. 2. Update [CHANGELOG.md](../CHANGELOG.md) to have the correct date and new version number, update [README.md](../README.md) to have the correct version numbers, and commit. -3. `mvn release:prepare` Use the new version number when prompted. -4. `mvn release:perform` +3. `mvn release:prepare -Dh3.github.artifacts.use=true` Use the new version number when prompted. +4. `mvn release:perform -Dh3.github.artifacts.use=true` 5. If this looks good, close and release the build in [Sonatype Nexus Manager](https://oss.sonatype.org/). 6. Update `CHANGELOG.md` to have an Unreleased section, and commit. The release is now done and development can resume from this point. ## Troubleshooting +### Dependencies for `pull-from-github.sh` + +* You should install the [Github CLI](https://cli.github.com) and authenticate with it first. You may need to use a personal access token (classic) with workflows scope. +* `jq` + ### gpg: signing failed: Inappropriate ioctl for device Per [StackOverflow](https://stackoverflow.com/questions/57591432/gpg-signing-failed-inappropriate-ioctl-for-device-on-macos-with-maven), run the following before `mvn release:perform`: diff --git a/pom.xml b/pom.xml index 6b828245..f8851db4 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ com.uber h3 jar - 4.0.3-SNAPSHOT + 4.1.0 h3 https://github.com/uber/h3-java Java bindings for H3, a hierarchical hexagonal geospatial indexing system. @@ -40,7 +40,7 @@ scm:git:git://github.com/uber/h3-java.git scm:git:ssh://git@github.com/uber/h3-java.git http://github.com/uber/h3-java/tree/master - HEAD + v4.1.0 @@ -72,6 +72,8 @@ true false 20210624-de7b1b0 + false + @@ -112,7 +114,7 @@ org.codehaus.mojo exec-maven-plugin - 1.1.1 + 3.1.0 @@ -127,7 +129,7 @@ -ExecutionPolicy Bypass -File - ${basedir}/src/main/c/h3-java/build-h3-windows.ps1 + ${project.basedir}/src/main/c/h3-java/build-h3-windows.ps1 ${h3.git.remote} ${h3.git.reference} @@ -203,7 +205,7 @@ - ${basedir}/h3version.properties + ${project.basedir}/h3version.properties @@ -223,7 +225,7 @@ -h - ${basedir}/src/main/c/h3-java/src + ${project.basedir}/src/main/c/h3-java/src @@ -250,7 +252,7 @@ org.codehaus.mojo exec-maven-plugin - 1.1.1 + 3.1.0 build-h3-c @@ -261,13 +263,15 @@ exec - ${basedir}/src/main/c/h3-java/build-h3.sh + ${project.basedir}/src/main/c/h3-java/build-h3.sh ${h3.git.remote} ${h3.git.reference} ${h3.use.docker} ${h3.system.prune} ${h3.dockcross.tag} + ${h3.github.artifacts.use} + ${h3.github.artifacts.by_run} @@ -288,7 +292,7 @@ ${project.build.outputDirectory} - ${basedir}/src/main/resources + ${project.basedir}/src/main/resources false diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 2f9fd936..741a0234 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -15,16 +15,21 @@ # limitations under the License. # -# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] -# git-remote - The git remote to pull from. An existing cloned repository -# will not be deleted if a new remote is specified. -# git-ref - Specific git ref of H3 to build. -# use-docker - "true" to perform cross compilation via Docker, "false" to -# skip that step. -# system-prune - If use-docker is true and this argument is true, Docker -# system prune will be run after each step -# (i.e. for disk space constrained environments like CI) -# dockcross-tag - Tag name for dockcross +# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] [github-artifacts] [github-artifacts-ref] +# git-remote - The git remote to pull from. An existing cloned repository +# will not be deleted if a new remote is specified. +# git-ref - Specific git ref of H3 to build. +# use-docker - "true" to perform cross compilation via Docker, "false" to +# skip that step. +# system-prune - If use-docker is true and this argument is true, Docker +# system prune will be run after each step +# (i.e. for disk space constrained environments like CI) +# dockcross-tag - Tag name for dockcross +# github-artifacts - When set, all build artifacts are retrieved from Github +# Actions artifacts rather than built locally (overrides +# all other settings.) +# github-artifacts-run - When set, assume the artifacts have already been +# downloaded. # # This script downloads H3, builds H3 and the H3-Java native library, and # cross compiles via Docker. @@ -39,6 +44,13 @@ GIT_REVISION=$2 USE_DOCKER=$3 SYSTEM_PRUNE=$4 DOCKCROSS_TAG=$5 +GITHUB_ARTIFACTS=$6 +GITHUB_ARTIFACTS_RUN=$7 + +if $GITHUB_ARTIFACTS; then + src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_RUN" + # The build of the library below is still needed for the binding-functions file +fi echo Downloading H3 from "$GIT_REMOTE" @@ -90,6 +102,11 @@ cp h3-java-build/build/binding-functions . popd # target +if $GITHUB_ARTIFACTS; then + # Nothing more is needed than the binding-functions to stop now. + exit 0 +fi + # Copy the built artifact for this platform. case "$(uname -sm)" in "Linux x86_64") LIBRARY_DIR=linux-x64 ;; diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh new file mode 100755 index 00000000..4b26ce01 --- /dev/null +++ b/src/main/c/h3-java/pull-from-github.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Copyright 2023 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Retrieves built artifacts from Github. The purpose of this script is +# to enable deployment from any system, regardless of whether it can +# run the particular cross compiling steps needed to build native shared +# objects. Some of the native shared objects require particular Docker +# setups, or indeed running on Mac OS, which is difficult to maintain. +# +# You must install the Github CLI and authenticate with a personal access +# token (classic) with workflows scope to use this. https://cli.github.com/ +# This script also requires `jq`. +# +# This script expects to be run from the project's base directory (where +# pom.xml is) as part of the Maven build process. + +set -eox pipefail + +GITHUB_ARTIFACTS_RUN=$1 + +EXTRACT_TO=src/main/resources + +if [ -z "$GITHUB_ARTIFACTS_RUN" ]; then + mkdir -p target + pushd target + + ARTIFACTS_LIST=$(gh api \ + -H "Accept: application/vnd.github+json" \ + /repos/{owner}/{repo}/actions/artifacts) + + echo "downloading artifacts for run $GITHUB_ARTIFACTS_RUN" + TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \ + | jq ".artifacts[] | select(.workflow_run.id == \"$GITHUB_ARTIFACTS_RUN\")") + + echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do + ARTIFACT_NAME=$(echo $artifactline | jq -r .name) + ARTIFACT_ID=$(echo $artifactline | jq .id) + echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID" + gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip" + unzip -o "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO" + done + + popd +fi + +echo Checking that expected images are present: + +for image in android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \ + linux-ppc64le linux-x64 linux-x86; do + if [ -f "$EXTRACT_TO/$image/libh3-java.so" ]; then + echo "$image" exists + else + echo "$image" missing! + exit 1 + fi +done + +for image in darwin-x64 darwin-arm64; do + if [ -f "$EXTRACT_TO/$image/libh3-java.dylib" ]; then + echo "$image" exists + else + echo "$image" missing! + exit 1 + fi +done + +for image in windows-x64 windows-x86; do + if [ -f "$EXTRACT_TO/$image/libh3-java.dll" ]; then + echo "$image" exists + else + echo "$image" missing! + exit 1 + fi +done