From cc2f9bf83eb93804689c8da4e6839d70e8ee9543 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 28 Oct 2026 20:41:12 +0000 Subject: [PATCH 1/5] Build with a uv-managed environment from make Make creates .venv with uv and installs all requirements files. A stamp file reinstalls only when a requirements file changes. Set VENV= to use tools on PATH (CI does this). Assisted-by: Claude Code:claude-opus-5-5 --- .github/workflows/pages.yml | 2 +- .github/workflows/test.yml | 2 +- .gitignore | 1 + CONTRIBUTING.md | 17 +++++++++++++---- Makefile | 33 +++++++++++++++++++++++++-------- _config.yml | 1 + 6 files changed, 42 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 062315a8f..3eb9175dd 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -31,7 +31,7 @@ jobs: - name: "Build HTML" run: | - make html + make html VENV= echo -n 'lectures.scientific-python.org' > _build/html/CNAME touch _build/html/.nojekyll diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 821eded8a..f5e3c3e86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,4 +37,4 @@ jobs: export JUPYTER_PLATFORM_DIRS=1 jupyter --paths - make test + make test VENV= diff --git a/.gitignore b/.gitignore index 4474e2447..f49401649 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,4 @@ advanced/advanced_numpy/test_red.png intro/language/junk.txt intro/language/test.pkl joblib/ +.venv/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 94eb0e598..fb627b7a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ To generate the html output for on-screen display, Type: make html ``` -the generated html files can be found in `build/html` +the generated html files can be found in `_build/html` The first build takes a long time, but information is cached and subsequent builds will be faster. @@ -62,11 +62,20 @@ Build requirements are listed in the ``` -Ensure that you have a [virtual environment](https://docs.python.org/3/library/venv.html) or conda environment -set up, then install requirements with: +With [uv](https://docs.astral.sh/uv/) installed, `make` creates a virtual +environment in `.venv` and installs all requirements into it. Make reinstalls +only when a requirements file changes. + +To use your own environment instead, install the requirements: + +``` +pip install -r build_requirements.txt +``` + +then tell `make` to use the tools on your `PATH`: ``` -pip install -r requirements.txt +make html VENV= ``` Note that you will also need the following system packages: diff --git a/Makefile b/Makefile index 79b13fccb..5213de1fe 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,23 @@ +# Set VENV= (empty) to use tools already on PATH instead of a uv environment. +VENV ?= .venv PYTHON ?= python -PIP_INSTALL_CMD ?= $(PYTHON) -m pip install BUILD_DIR=_build/html JL_DIR=_build/jl +REQUIREMENTS = requirements.txt build_requirements.txt test_requirements.txt jl-build-requirements.txt -html: +ifdef VENV +export VIRTUAL_ENV := $(abspath $(VENV)) +export PATH := $(VIRTUAL_ENV)/bin:$(PATH) +ENV_STAMP := $(VENV)/.installed +endif + +html: $(ENV_STAMP) # Check for ipynb files in source (should all be text - .md or .Rmd). if compgen -G "*.ipynb" 2> /dev/null; then (echo "ipynb files" && exit 1); fi jupyter-book build -W . -jl: +jl: $(ENV_STAMP) # Jupyter-lite files for book build. - $(PIP_INSTALL_CMD) -r jl-build-requirements.txt rm -rf $(JL_DIR) mkdir $(JL_DIR) cp -r data images $(JL_DIR) @@ -20,7 +27,7 @@ jl: --output-dir $(BUILD_DIR)/interact \ --lite-dir $(JL_DIR) -lint: +lint: $(ENV_STAMP) pre-commit run --all-files --show-diff-on-failure --color always web: html jl @@ -36,9 +43,19 @@ clean: rm-ipynb rm-ipynb: find . -name "*.ipynb" -exec rm {} \; -test: +test: $(ENV_STAMP) pytest . -compare-optimizers: +compare-optimizers: $(ENV_STAMP) ( cd advanced/mathematical_optimization/helper && \ - python compare_optimizers.py ) + $(PYTHON) compare_optimizers.py ) + +# Reinstall only when a requirements file is newer than the stamp. +$(ENV_STAMP): $(REQUIREMENTS) + uv venv --allow-existing $(VENV) + uv pip install $(addprefix -r ,$(REQUIREMENTS)) + touch $@ + +env: $(ENV_STAMP) + +.PHONY: html jl lint web github clean rm-ipynb test compare-optimizers env diff --git a/_config.yml b/_config.yml index 8683e0134..f94175ff0 100644 --- a/_config.yml +++ b/_config.yml @@ -27,6 +27,7 @@ exclude_patterns: - data/LICENSE.txt - .pytest_cache/* - .ipynb_checkpoints/* + - .venv/** - todo.md html: From 11a8fa34b1695fa65a2b3234d2d5e7366860e65b Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 28 Oct 2026 23:46:44 +0000 Subject: [PATCH 2/5] CI: install dependencies with uv through make Use setup-uv in GitHub workflows and install uv on CircleCI. Assisted-by: Claude Code:claude-opus-5-5 --- .circleci/config.yml | 15 ++++++++------- .github/workflows/lint.yml | 12 +++--------- .github/workflows/pages.yml | 13 ++++--------- .github/workflows/test.yml | 16 +++------------- Makefile | 2 ++ 5 files changed, 20 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b46fdfa0..5e87b9152 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,19 +10,20 @@ jobs: - restore_cache: keys: - - pip-cache-v1 + - uv-cache-v1 + + - run: + name: Install uv + command: pip install --user uv - run: name: Install Python dependencies - command: | - pip install --upgrade --user pip - pip install --user -r build_requirements.txt - pip list + command: make env - save_cache: - key: pip-cache-v1 + key: uv-cache-v1 paths: - - ~/.cache/pip + - ~/.cache/uv - run: name: Build docs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 05f1ce117..7a13c8144 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,16 +19,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Set up uv with Python ${{ matrix.python-version }} + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: python-version: ${{ matrix.python-version }} - - name: Install packages - run: | - pip install --upgrade pip wheel setuptools - pip install -r build_requirements.txt - pip list - - name: Lint - run: pre-commit run --all-files --show-diff-on-failure --color always + run: make lint diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 3eb9175dd..c5bcbe098 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.12 - uses: actions/setup-python@v5 + - name: Set up uv with Python 3.12 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: - python-version: 3.12 + python-version: "3.12" - name: Install TeX run: | @@ -24,14 +24,9 @@ jobs: texlive-fonts-extra \ texlive-extra-utils - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip wheel setuptools - python -m pip install -r build_requirements.txt - - name: "Build HTML" run: | - make html VENV= + make html echo -n 'lectures.scientific-python.org' > _build/html/CNAME touch _build/html/.nojekyll diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5e3c3e86..328b27e20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,20 +21,10 @@ jobs: python-version: ["3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Set up uv with Python ${{ matrix.python-version }} + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: python-version: ${{ matrix.python-version }} - - name: Install packages - run: | - python -m pip install -r test_requirements.txt - python -m pip list - - name: Test - run: | - # Avoid deprecation error. - export JUPYTER_PLATFORM_DIRS=1 - jupyter --paths - - make test VENV= + run: make test diff --git a/Makefile b/Makefile index 5213de1fe..3903f13d9 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ clean: rm-ipynb rm-ipynb: find . -name "*.ipynb" -exec rm {} \; +# Avoid a Jupyter deprecation warning, which pytest treats as an error. +test: export JUPYTER_PLATFORM_DIRS = 1 test: $(ENV_STAMP) pytest . From 6797b58945503db2ead2bf37459d73d4ff50d40e Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Wed, 28 Oct 2026 23:47:50 +0000 Subject: [PATCH 3/5] Fix mypy errors in process_notebooks.py Assisted-by: Claude Code:claude-opus-5-5 --- _scripts/process_notebooks.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/_scripts/process_notebooks.py b/_scripts/process_notebooks.py index 66549a534..94b56e8b1 100755 --- a/_scripts/process_notebooks.py +++ b/_scripts/process_notebooks.py @@ -131,15 +131,27 @@ def get_admonition_lines(nb_text, nb_path): lines = nb_text.splitlines() n_lines = len(lines) admonition_lines = [] - for admonition in doc.findall(dun.Admonition): + admonitions = ( + node for node in doc.findall(dun.Element) if isinstance(node, dun.Admonition) + ) + for admonition in admonitions: + assert admonition.line is not None start_line = admonition.line - 1 # Find first node of subsequent doctree. node0 = next( - admonition.findall(include_self=False, descend=False, ascend=True), None + admonition.findall( + dun.Element, include_self=False, descend=False, ascend=True + ), + None, ) # There can be a system_message as next node, in which case the correct # line is in the 'line' attribute. - last_line = node0.get("line", node0.line) - 2 if node0 else n_lines - 1 + if node0: + next_line = node0.get("line", node0.line) + assert next_line is not None + last_line = next_line - 2 + else: + last_line = n_lines - 1 for end_line in range(last_line, start_line + 1, -1): if _END_DIV_RE.match(lines[end_line]): break From 838296e5b620e50fa974ff16d2bf2e957c4cc3e3 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Fri, 30 Oct 2026 20:18:14 +0000 Subject: [PATCH 4/5] Pass absolute paths to jupyter lite build jupyterlite-core resolves --contents relative to --lite-dir. Assisted-by: Claude Code:claude-opus-5-5 --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3903f13d9..ee178f335 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ jl: $(ENV_STAMP) cp -r data images $(JL_DIR) $(PYTHON) _scripts/process_notebooks.py $(JL_DIR) $(PYTHON) -m jupyter lite build \ - --contents $(JL_DIR) \ - --output-dir $(BUILD_DIR)/interact \ - --lite-dir $(JL_DIR) + --contents $(abspath $(JL_DIR)) \ + --output-dir $(abspath $(BUILD_DIR))/interact \ + --lite-dir $(abspath $(JL_DIR)) lint: $(ENV_STAMP) pre-commit run --all-files --show-diff-on-failure --color always From 5ee7d8f12d191c0e0dff2682f0b130adc45db2d2 Mon Sep 17 00:00:00 2001 From: Stefan van der Walt Date: Fri, 30 Oct 2026 20:19:57 +0000 Subject: [PATCH 5/5] Call venv tools by path for Make 3.81 on macOS Assisted-by: Claude Code:claude-opus-5-5 --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ee178f335..19e75646e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Set VENV= (empty) to use tools already on PATH instead of a uv environment. VENV ?= .venv -PYTHON ?= python +PYTHON ?= $(BIN)python BUILD_DIR=_build/html JL_DIR=_build/jl REQUIREMENTS = requirements.txt build_requirements.txt test_requirements.txt jl-build-requirements.txt @@ -8,13 +8,15 @@ REQUIREMENTS = requirements.txt build_requirements.txt test_requirements.txt jl- ifdef VENV export VIRTUAL_ENV := $(abspath $(VENV)) export PATH := $(VIRTUAL_ENV)/bin:$(PATH) +# Make 3.81 (macOS) ignores the exported PATH when it runs a command directly. +BIN := $(VIRTUAL_ENV)/bin/ ENV_STAMP := $(VENV)/.installed endif html: $(ENV_STAMP) # Check for ipynb files in source (should all be text - .md or .Rmd). if compgen -G "*.ipynb" 2> /dev/null; then (echo "ipynb files" && exit 1); fi - jupyter-book build -W . + $(BIN)jupyter-book build -W . jl: $(ENV_STAMP) # Jupyter-lite files for book build. @@ -28,12 +30,12 @@ jl: $(ENV_STAMP) --lite-dir $(abspath $(JL_DIR)) lint: $(ENV_STAMP) - pre-commit run --all-files --show-diff-on-failure --color always + $(BIN)pre-commit run --all-files --show-diff-on-failure --color always web: html jl github: web - ghp-import -n _build/html -p -f + $(BIN)ghp-import -n _build/html -p -f clean: rm-ipynb rm -rf _build @@ -46,7 +48,7 @@ rm-ipynb: # Avoid a Jupyter deprecation warning, which pytest treats as an error. test: export JUPYTER_PLATFORM_DIRS = 1 test: $(ENV_STAMP) - pytest . + $(BIN)pytest . compare-optimizers: $(ENV_STAMP) ( cd advanced/mathematical_optimization/helper && \