Skip to content
CI

RFC for LoadWith and SaveWith implementation for DocumentProvider #629

Sign in to view logs

RFC for LoadWith and SaveWith implementation for DocumentProvider

RFC for LoadWith and SaveWith implementation for DocumentProvider #629

Workflow file for this run

.github/workflows/ci.yml at ba15b33
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license.
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
name: CI
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
env:
# Pass additional arguments to cargo using `--config build.rustflags=["...", "..."]`.
#
# This ensures that we do not overwrite the parameters in `.cargo/config.toml` unnecessarily.
RUST_CONFIG: 'build.rustflags=["-Dwarnings"]'
RUST_BACKTRACE: 1
# The features we want to explicitly test. For example, the `flatbuffers-build` feature
# of `diskann-quantization` requires additional setup and so must not be included by default.
DISKANN_FEATURES: "virtual_storage,bf_tree,spherical-quantization,product-quantization,tracing,experimental_diversity_search,disk-index,flatbuffers,linalg,codegen"
# Use the Rust version specified in rust-toolchain.toml
rust_stable: "1.92"
defaults:
run:
shell: bash
permissions:
contents: read
jobs:
# Basic checks that must pass before we kick off more expensive tests.
basics:
name: basic checks
runs-on: ubuntu-latest
needs:
- fmt
- clippy-default-features
- clippy-features
- clippy-no-default-features
# TODO: Re-enable docs check later
# - docs
steps:
- run: exit 0
fmt:
name: format check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: rustfmt
- uses: Swatinem/rust-cache@v2
# Check fmt
- name: "cargo fmt --check"
run: |
if ! cargo fmt --all --check; then
printf "Please run \`cargo fmt --all\` to fix rustfmt errors.\n" >&2
exit 1
fi
clippy-default-features:
strategy:
matrix:
runner:
- ubuntu-latest
- ubuntu-24.04-arm
- windows-latest
fail-fast: false
name: clippy-default-features (${{ matrix.runner }})
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: clippy
- uses: Swatinem/rust-cache@v2
- name: "clippy --workspace --all-targets"
run: cargo clippy --locked --workspace --all-targets --no-deps --config "$RUST_CONFIG" -- -Dwarnings
clippy-features:
strategy:
matrix:
runner:
- ubuntu-latest
- ubuntu-24.04-arm
- windows-latest
fail-fast: false
name: clippy-features (${{ matrix.runner }})
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: clippy
- uses: Swatinem/rust-cache@v2
- name: "clippy --workspace --all-targets"
run: |
set -euxo pipefail
cargo clippy --locked --workspace \
--all-targets \
--no-deps \
--features ${{ env.DISKANN_FEATURES }} \
--config "$RUST_CONFIG" \
-- -Dwarnings
clippy-no-default-features:
name: clippy (no default features)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
lfs: true
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: clippy
- uses: Swatinem/rust-cache@v2
- name: Clippy - select crates (no default features)
run: |
set -euxo pipefail
crates=(
# Core crates
diskann
diskann-providers
diskann-disk
diskann-quantization
diskann-utils
# Benchmark/tools crates
diskann-benchmark-core
diskann-benchmark-runner
diskann-benchmark
diskann-tools
)
for crate in "${crates[@]}"; do
cargo clippy --locked --package "$crate" \
--no-default-features \
--profile ci \
--no-deps \
--config "$RUST_CONFIG" \
-- -Dwarnings
done
codeql:
name: CodeQL security analysis
runs-on: ubuntu-latest
permissions:
contents: read
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: rust
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- uses: Swatinem/rust-cache@v2
- name: Build workspace
run: cargo build --workspace --locked --profile ci
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:rust"
# TODO: Re-enable docs check later
# docs:
# name: docs
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - name: Install Rust ${{ env.rust_stable }}
# uses: dtolnay/rust-toolchain@master
# with:
# toolchain: ${{ env.rust_stable }}
# - uses: Swatinem/rust-cache@v2
# - name: "doc --workspace --no-deps"
# run: cargo doc --workspace --no-deps --document-private-items
# env:
# RUSTDOCFLAGS: -Dwarnings
# This is a quick test to ensure that micro-architecture detection and dispatching works.
#
# The crate `diskann-wide` is the primary driver for this mechanism.
#
# We test by:
#
# 1. Compiling the test binaries specifically for the old `x86-64` CPU target to ensure
# the compiler does not emit `x86-64-v3` instructions.
#
# 2. We use QEMU to emulate an older CPU. QEMU will abort if an unrecognized instruction
# is hit.
#
# The shell command for running tests is as follows:
#
# 1. `find ./target/**/deps`: Search for the generated artifacts from the test build.
#
# 2. `grep -e "dispatch-[a-f0-9]*$"`: Find a binary called `dispatch-a38d0...` that does
# not end in ".d". In other words, the hash goes all the way to the end of the file name.
#
# This means it's a test executable and not a library.
#
# 3. `xargs -n1`: Take the newline delimited executable and call the following command
# once for each argument.
#
# 4. `qemu-x86_64 -cpu Nehalem`: Run the qemu binary using `Nehalem` as the target CPU.
# This is a very old CPU model that will crash if any AVX/AVX2 instructions are executed.
#
# TODO: Enable whole test suite to run under QEMU. There are a few tests that need fixing.
qemu:
needs: basics
name: qemu-tests
runs-on: ubuntu-latest
env:
# For this test - we explicitly overwrite the flags in `.cargo/config.toml` because
# we want to ensure the `x86-64` target CPU.
RUSTFLAGS: "-Dwarnings -Ctarget-cpu=x86-64"
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Install QEMU user-mode
run: sudo apt-get update && sudo apt-get install -y qemu-user
- name: "QEMU Tests"
run: |
set -euxo pipefail
cargo test --package diskann-wide --locked
find ./target -type f -regex ".*/deps/dispatch-[a-f0-9]*$" | xargs -n1 qemu-x86_64 -cpu Nehalem
find ./target -type f -regex ".*/deps/float16_conversion-[a-f0-9]*$" | xargs -n1 qemu-x86_64 -cpu Nehalem
test-workspace:
needs: basics
name: test workspace
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- windows-latest
- ubuntu-latest
- ubuntu-24.04-arm
- macos-latest
steps:
- uses: actions/checkout@v4
with:
lfs: true
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install cargo-nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- uses: Swatinem/rust-cache@v2
- name: test workspace with nextest
run: |
set -euxo pipefail
cargo nextest run --locked --workspace --cargo-profile ci --config "$RUST_CONFIG"
cargo test --locked --doc --workspace --profile ci --config "$RUST_CONFIG"
test-workspace-features:
needs: basics
name: test workspace (all features)
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- windows-latest
- ubuntu-latest
- macos-latest
steps:
- uses: actions/checkout@v4
with:
lfs: true
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install cargo-nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- uses: Swatinem/rust-cache@v2
- name: test workspace with nextest
run: |
set -euxo pipefail
cargo nextest run --locked --workspace \
--cargo-profile ci \
--config "$RUST_CONFIG" \
--features ${{ env.DISKANN_FEATURES }}
cargo test --locked --doc --workspace --profile ci --config "$RUST_CONFIG"
coverage:
needs: basics
name: code coverage
runs-on: ubuntu-latest
if: github.actor != 'dependabot[bot]'
steps:
- uses: actions/checkout@v4
with:
lfs: true
fetch-depth: 0
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: llvm-tools-preview
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@v2
with:
tool: cargo-llvm-cov
- name: Install cargo-nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- uses: Swatinem/rust-cache@v2
- name: Generate code coverage
run: |
cargo llvm-cov nextest --locked --cargo-profile ci \
--config "$RUST_CONFIG" \
--workspace \
--lcov --output-path lcov.info
- name: Generate miri code coverage
env:
RUSTFLAGS: "--cfg=miri"
run: |
cargo +nightly llvm-cov nextest --locked --cargo-profile ci \
--package diskann-quantization \
--lcov --output-path lcov_miri.info
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
with:
files: lcov.info
fail_ci_if_error: true
flags: unittests
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Upload miri coverage to Codecov
uses: codecov/codecov-action@v4
with:
files: lcov_miri.info
fail_ci_if_error: true
flags: miri
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
miri:
needs: basics
name: miri-test
# This step is slow, so it only runs after a PR merge to avoid slowing down pre-merge checks.
if: ${{ github.event_name == 'push' }}
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v4
with:
lfs: true
- name: Install Rust nightly with miri
uses: dtolnay/rust-toolchain@master
with:
toolchain: nightly
components: miri
- name: Install cargo-nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- uses: Swatinem/rust-cache@v2
- name: miri
run: cargo +nightly miri nextest run --locked --package diskann-quantization
env:
MIRIFLAGS: -Zmiri-disable-isolation -Zmiri-strict-provenance