refactor(cache): faster lookup using list_repo_tree #558
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build documentation | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v[0-9]+.[0-9]+.[0-9]+' | |
| paths: | |
| - 'docs/source/**' | |
| - 'docs/assets/**' | |
| - 'notebooks/**' | |
| - 'optimum/**' | |
| - '.github/workflows/doc-build.yml' | |
| workflow_dispatch: | |
| jobs: | |
| build_documentation: | |
| runs-on: ubuntu-22.04 | |
| env: | |
| COMMIT_SHA: ${{ github.event.pull_request.head.sha }} | |
| PR_NUMBER: ${{ github.event.number }} | |
| EVENT_CONTEXT: ${{ toJSON(github.event) }} | |
| PR_CLONE_URL: ${{ github.event.pull_request.head.repo.clone_url }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '20' | |
| cache-dependency-path: "kit/package-lock.json" | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Neuronx runtime | |
| uses: ./.github/actions/install_neuronx_runtime | |
| - name: Prepare virtual environment | |
| uses: ./.github/actions/prepare_venv | |
| - name: Install optimum-neuron | |
| uses: ./.github/actions/install_optimum_neuron | |
| - name: Setup PATH | |
| run: echo "/home/ubuntu/.local/bin" >> $GITHUB_PATH | |
| - name: Set environment variables | |
| run: | | |
| cd optimum | |
| version=`echo "$(grep '^__version__ =' neuron/version.py | cut -d '=' -f 2- | xargs)"` | |
| if [[ $version == *.dev* ]] | |
| then | |
| echo "VERSION=main" >> $GITHUB_ENV | |
| else | |
| echo "VERSION=v$version" >> $GITHUB_ENV | |
| fi | |
| cd .. | |
| - name: Setup environment | |
| run: | | |
| source aws_neuron_venv_pytorch/bin/activate | |
| python -m ensurepip --upgrade | |
| python -m pip install --upgrade setuptools | |
| python -m pip install ".[quality, diffusers]" | |
| python -m pip install "git+https://github.com/huggingface/doc-builder.git" | |
| - name: Convert notebooks | |
| shell: bash | |
| run: | | |
| mkdir -p docs/source/inference_tutorials/ | |
| mkdir -p docs/source/training_tutorials/ | |
| source aws_neuron_venv_pytorch/bin/activate | |
| doc-builder notebook-to-mdx notebooks/text-classification/fine_tune_bert.ipynb --output_dir docs/source/training_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/sagemaker/deploy-llama-3-3-70b.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/sagemaker/deploy-mixtral-8x7b.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/sentence-transformers/sentence-transformers.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/text-generation/llama2-13b-chatbot.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/stable-diffusion/stable-diffusion-txt2img.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/stable-diffusion/stable-diffusion-xl-txt2img.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/text-generation/CodeLlama-7B-Compilation.ipynb --output_dir docs/source/inference_tutorials/ | |
| doc-builder notebook-to-mdx notebooks/feature-extraction/qwen_embedding.ipynb --output_dir docs/source/inference_tutorials/ | |
| - name: Make documentation | |
| shell: bash | |
| run: | | |
| source aws_neuron_venv_pytorch/bin/activate | |
| doc-builder build optimum.neuron docs/source/ \ | |
| --repo_name optimum-neuron \ | |
| --build_dir neuron-doc-build/ \ | |
| --version ${{ env.VERSION }} \ | |
| --version_tag_suffix "" \ | |
| --html \ | |
| --clean \ | |
| --notebook_dir docs/notebooks/ | |
| cd neuron-doc-build/ | |
| mv optimum.neuron optimum-neuron | |
| doc-builder push optimum-neuron --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum-neuron/commit/$COMMIT_SHA" --n_retries 5 |