upload artifacts

.github/workflows/tests.yml

@@ -45,6 +45,16 @@ jobs:
       - name: Format check for C
         run: git diff --exit-code
 
+      - uses: actions/upload-artifact@v3
+        name: Upload artifacts
+        if: ${{ matrix.java-version == 8 }}
+        with:
+          name: docker-built-shared-objects
+          path: |
+            src/main/resources/*/*.so
+            src/main/resources/*/*.dll
+          if-no-files-found: error 
+
   tests-new-dockcross:
     name: Dockcross ${{ matrix.dockcross-tag }} Java ${{ matrix.java-version }} ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -54,7 +64,7 @@ jobs:
         os: [ubuntu-latest]
         java-distribution: [adopt]
         java-version: [17]
-        dockcross-tag: ["20220906-e88a3ce"]
+        dockcross-tag: ["20220906-e88a3ce", "20230116-670f7f7"]
 
     steps:
       - uses: actions/[email protected]
@@ -109,6 +119,14 @@ jobs:
       - name: Tests
         run: mvn -B -V clean test site
 
+      - uses: actions/upload-artifact@v3
+        name: Upload Mac OS Artifacts
+        if: ${{ matrix.os == 'macos-latest' && matrix.java-version == 8 }}
+        with:
+          name: macos-built-shared-objects
+          path: src/main/resources/*/*.dylib
+          if-no-files-found: error 
+
   tests-coverage:
     name: Java (Coverage) ${{ matrix.java-version }} ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -143,3 +161,52 @@ jobs:
         env:
           CI_NAME: github
           COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} 
+
+  tests-use-built-artifacts:
+    name: Java (Built Artifacts) ${{ matrix.java-version }} ${{ matrix.os }}
+    needs:
+      - tests
+      - tests-no-docker
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java-distribution: [adopt]
+        java-version: [8]
+
+    steps:
+      - uses: actions/[email protected]
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-java@v2
+        with:
+          distribution: "${{ matrix.java-distribution }}"
+          java-version: "${{ matrix.java-version }}"
+
+      - uses: actions/cache@v2
+        id: maven-cache
+        with:
+          path: ~/.m2/
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Download Docker binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-built-shared-objects
+          path: src/main/resources/
+
+      - name: Download Mac binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: macos-built-shared-objects
+          path: src/main/resources/
+
+      - name: Download and test
+        run: |
+          mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.by_run=true
+        env:
+          GH_TOKEN: ${{ github.token }}

docs/releasing.md

@@ -8,13 +8,18 @@ Release builds should only be run on Mac OSX with Docker. This is needed so that
 
 1. Ensure you are on branch `master` and that the library is building correctly.
 2. Update [CHANGELOG.md](../CHANGELOG.md) to have the correct date and new version number, update [README.md](../README.md) to have the correct version numbers, and commit.
-3. `mvn release:prepare` Use the new version number when prompted.
-4. `mvn release:perform`
+3. `mvn release:prepare -Dh3.github.artifacts.use=true` Use the new version number when prompted.
+4. `mvn release:perform -Dh3.github.artifacts.use=true`
 5. If this looks good, close and release the build in [Sonatype Nexus Manager](https://oss.sonatype.org/).
 6. Update `CHANGELOG.md` to have an Unreleased section, and commit. The release is now done and development can resume from this point.
 
 ## Troubleshooting
 
+### Dependencies for `pull-from-github.sh`
+
+* You should install the [Github CLI](https://cli.github.com) and authenticate with it first. You may need to use a personal access token (classic) with workflows scope.
+* `jq`
+
 ### gpg: signing failed: Inappropriate ioctl for device
 
 Per [StackOverflow](https://stackoverflow.com/questions/57591432/gpg-signing-failed-inappropriate-ioctl-for-device-on-macos-with-maven), run the following before `mvn release:perform`:

pom.xml

@@ -18,7 +18,7 @@
     <groupId>com.uber</groupId>
     <artifactId>h3</artifactId>
     <packaging>jar</packaging>
-    <version>4.0.3-SNAPSHOT</version>
+    <version>4.1.0</version>
     <name>h3</name>
     <url>https://github.com/uber/h3-java</url>
     <description>Java bindings for H3, a hierarchical hexagonal geospatial indexing system.</description>
@@ -40,7 +40,7 @@
         <connection>scm:git:git://github.com/uber/h3-java.git</connection>
         <developerConnection>scm:git:ssh://[email protected]/uber/h3-java.git</developerConnection>
         <url>http://github.com/uber/h3-java/tree/master</url>
-      <tag>HEAD</tag>
+      <tag>v4.1.0</tag>
   </scm>
 
     <developers>
@@ -72,6 +72,8 @@
         <h3.use.docker>true</h3.use.docker>
         <h3.system.prune>false</h3.system.prune>
         <h3.dockcross.tag>20210624-de7b1b0</h3.dockcross.tag>
+        <h3.github.artifacts.use>false</h3.github.artifacts.use>
+        <h3.github.artifacts.by_run />
         <!-- Used for passing additional arguments to surefire when using JaCoCo -->
         <h3.additional.argLine />
 
@@ -112,7 +114,7 @@
                     <plugin>
                         <groupId>org.codehaus.mojo</groupId>
                         <artifactId>exec-maven-plugin</artifactId>
-                        <version>1.1.1</version>
+                        <version>3.1.0</version>
                         <executions>
                             <execution>
                                 <!-- Override when building on Windows to use PowerShell & MSVC -->
@@ -127,7 +129,7 @@
                                         <argument>-ExecutionPolicy</argument>
                                         <argument>Bypass</argument>
                                         <argument>-File</argument>
-                                        <argument>${basedir}/src/main/c/h3-java/build-h3-windows.ps1</argument>
+                                        <argument>${project.basedir}/src/main/c/h3-java/build-h3-windows.ps1</argument>
                                         <argument>${h3.git.remote}</argument>
                                         <argument>${h3.git.reference}</argument>
                                     </arguments>
@@ -203,7 +205,7 @@
                         </goals>
                         <configuration>
                             <files>
-                                <file>${basedir}/h3version.properties</file>
+                                <file>${project.basedir}/h3version.properties</file>
                             </files>
                         </configuration>
                     </execution>
@@ -223,7 +225,7 @@
 
                         <!-- Generate the native header file -->
                         <arg>-h</arg>
-                        <arg>${basedir}/src/main/c/h3-java/src</arg>
+                        <arg>${project.basedir}/src/main/c/h3-java/src</arg>
                     </compilerArgs>
                 </configuration>
             </plugin>
@@ -250,7 +252,7 @@
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>exec-maven-plugin</artifactId>
-                <version>1.1.1</version>
+                <version>3.1.0</version>
                 <executions>
                     <execution>
                         <id>build-h3-c</id>
@@ -261,13 +263,15 @@
                             <goal>exec</goal>
                         </goals>
                         <configuration>
-                            <executable>${basedir}/src/main/c/h3-java/build-h3.sh</executable>
+                            <executable>${project.basedir}/src/main/c/h3-java/build-h3.sh</executable>
                             <arguments>
                                 <argument>${h3.git.remote}</argument>
                                 <argument>${h3.git.reference}</argument>
                                 <argument>${h3.use.docker}</argument>
                                 <argument>${h3.system.prune}</argument>
                                 <argument>${h3.dockcross.tag}</argument>
+                                <argument>${h3.github.artifacts.use}</argument>
+                                <argument>${h3.github.artifacts.by_run}</argument>
                             </arguments>
                         </configuration>
                     </execution>
@@ -288,7 +292,7 @@
                             <outputDirectory>${project.build.outputDirectory}</outputDirectory>
                             <resources>
                                 <resource>
-                                    <directory>${basedir}/src/main/resources</directory>
+                                    <directory>${project.basedir}/src/main/resources</directory>
                                     <filtering>false</filtering>
                                 </resource>
                             </resources>

src/main/c/h3-java/build-h3.sh

@@ -15,16 +15,21 @@
 # limitations under the License.
 #
 
-# Arguments: [git-remote] [git-ref] [use-docker] [remove-images]
-# git-remote    - The git remote to pull from. An existing cloned repository
-#                 will not be deleted if a new remote is specified.
-# git-ref       - Specific git ref of H3 to build.
-# use-docker    - "true" to perform cross compilation via Docker, "false" to
-#                 skip that step.
-# system-prune  - If use-docker is true and this argument is true, Docker
-#                 system prune will be run after each step
-#                 (i.e. for disk space constrained environments like CI)
-# dockcross-tag - Tag name for dockcross
+# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] [github-artifacts] [github-artifacts-ref]
+# git-remote       - The git remote to pull from. An existing cloned repository
+#                    will not be deleted if a new remote is specified.
+# git-ref          - Specific git ref of H3 to build.
+# use-docker       - "true" to perform cross compilation via Docker, "false" to
+#                    skip that step.
+# system-prune     - If use-docker is true and this argument is true, Docker
+#                    system prune will be run after each step
+#                    (i.e. for disk space constrained environments like CI)
+# dockcross-tag    - Tag name for dockcross
+# github-artifacts - When set, all build artifacts are retrieved from Github
+#                    Actions artifacts rather than built locally (overrides
+#                    all other settings.)
+# github-artifacts-run - When set, assume the artifacts have already been
+#                        downloaded.
 #
 # This script downloads H3, builds H3 and the H3-Java native library, and
 # cross compiles via Docker.
@@ -39,6 +44,13 @@ GIT_REVISION=$2
 USE_DOCKER=$3
 SYSTEM_PRUNE=$4
 DOCKCROSS_TAG=$5
+GITHUB_ARTIFACTS=$6
+GITHUB_ARTIFACTS_RUN=$7
+
+if $GITHUB_ARTIFACTS; then
+    src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_RUN"
+    # The build of the library below is still needed for the binding-functions file
+fi
 
 echo Downloading H3 from "$GIT_REMOTE"
 
@@ -90,6 +102,11 @@ cp h3-java-build/build/binding-functions .
 
 popd # target
 
+if $GITHUB_ARTIFACTS; then
+    # Nothing more is needed than the binding-functions to stop now.
+    exit 0
+fi
+
 # Copy the built artifact for this platform.
 case "$(uname -sm)" in
     "Linux x86_64")  LIBRARY_DIR=linux-x64 ;;

src/main/c/h3-java/pull-from-github.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# Copyright 2023 Uber Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Retrieves built artifacts from Github. The purpose of this script is
+# to enable deployment from any system, regardless of whether it can
+# run the particular cross compiling steps needed to build native shared
+# objects. Some of the native shared objects require particular Docker
+# setups, or indeed running on Mac OS, which is difficult to maintain.
+#
+# You must install the Github CLI and authenticate with a personal access
+# token (classic) with workflows scope to use this. https://cli.github.com/
+# This script also requires `jq`.
+#
+# This script expects to be run from the project's base directory (where
+# pom.xml is) as part of the Maven build process.
+
+set -eox pipefail
+
+GITHUB_ARTIFACTS_RUN=$1
+
+EXTRACT_TO=src/main/resources
+
+if [ -z "$GITHUB_ARTIFACTS_RUN" ]; then
+    mkdir -p target
+    pushd target
+
+    ARTIFACTS_LIST=$(gh api \
+    -H "Accept: application/vnd.github+json" \
+    /repos/{owner}/{repo}/actions/artifacts)
+
+    echo "downloading artifacts for run $GITHUB_ARTIFACTS_RUN"
+    TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \
+        | jq ".artifacts[] | select(.workflow_run.id == \"$GITHUB_ARTIFACTS_RUN\")")
+
+    echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do
+        ARTIFACT_NAME=$(echo $artifactline | jq -r .name)
+        ARTIFACT_ID=$(echo $artifactline | jq .id)
+        echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID"
+        gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip"
+        unzip -o "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO"
+    done
+
+    popd
+fi
+
+echo Checking that expected images are present:
+
+for image in android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \
+    linux-ppc64le linux-x64 linux-x86; do
+    if [ -f "$EXTRACT_TO/$image/libh3-java.so" ]; then
+        echo "$image" exists
+    else
+        echo "$image" missing!
+        exit 1
+    fi
+done
+
+for image in darwin-x64 darwin-arm64; do
+    if [ -f "$EXTRACT_TO/$image/libh3-java.dylib" ]; then
+        echo "$image" exists
+    else
+        echo "$image" missing!
+        exit 1
+    fi
+done
+
+for image in windows-x64 windows-x86; do
+    if [ -f "$EXTRACT_TO/$image/libh3-java.dll" ]; then
+        echo "$image" exists
+    else
+        echo "$image" missing!
+        exit 1
+    fi
+done