diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
index 7482934124e5b..c7d9c3d45fde1 100644
--- a/.github/workflows/build-docker.yml
+++ b/.github/workflows/build-docker.yml
@@ -81,6 +81,27 @@ jobs:
           platform=${{ matrix.platform }}
           echo "PLATFORM_TUPLE=${platform//\//-}" >> $GITHUB_ENV
 
+      - name: Docker build caches
+        uses: actions/cache@v4
+        id: docker-build-caches
+        with:
+          path: |
+            var-cache-apt
+            var-lib-apt
+            tool-caches
+          key: docker-caches-${{ matrix.platform }}-${{ hashFiles('Dockerfile') }}
+
+      - name: Inject build caches into docker
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          cache-map: |
+            {
+              "var-cache-apt": "/var/cache/apt",
+              "var-lib-apt": "/var/lib/apt",
+              "tool-caches": {"target": "/buildkit-cache", "id": "tool-caches"}
+            }
+          skip-extraction: ${{ steps.docker-build-caches.outputs.cache-hit }}
+
       # Adapted from https://docs.docker.com/build/ci/github-actions/multi-platform/
       - name: Build and push by digest
         id: build
diff --git a/Dockerfile b/Dockerfile
index 744909a181f62..b2b2ded717f96 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,19 +1,32 @@
 FROM --platform=$BUILDPLATFORM ubuntu AS build
-ENV HOME="/root"
+ENV HOME="/root" \
+  # Place tool-specific caches in the buildkit tool cache.
+  CARGO_HOME=/buildkit-cache/cargo \
+  CARGO_ZIGBUILD_CACHE_DIR=/buildkit-cache/cargo-zigbuild \
+  PIP_CACHE_DIR=/buildkit-cache/pip \
+  RUSTUP_HOME=/buildkit-cache/rustup \
+  ZIG_GLOBAL_CACHE_DIR=/buildkit-cache/zig
 WORKDIR $HOME
 
-RUN apt update \
+RUN \
+  --mount=type=cache,target=/var/cache/apt,sharing=locked \
+  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  # remove the default docker-specific apt config that auto-deletes /var/apt/cache archives
+  rm -f /etc/apt/apt.conf.d/docker-clean && \
+  # and configure apt-get to keep downloaded archives in the cache
+  echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
+  apt update \
   && apt install -y --no-install-recommends \
   build-essential \
   curl \
   python3-venv \
-  cmake \
-  && apt clean \
-  && rm -rf /var/lib/apt/lists/*
+  cmake
 
 # Setup zig as cross compiling linker
 RUN python3 -m venv $HOME/.venv
-RUN .venv/bin/pip install cargo-zigbuild
+RUN \
+  --mount=type=cache,target=/buildkit-cache,id="tool-caches" \
+  .venv/bin/pip install cargo-zigbuild
 ENV PATH="$HOME/.venv/bin:$PATH"
 
 # Install rust
@@ -25,21 +38,32 @@ RUN case "$TARGETPLATFORM" in \
   esac
 
 # Update rustup whenever we bump the rust version
+ENV PATH="$CARGO_HOME/bin:$PATH"
 COPY rust-toolchain.toml rust-toolchain.toml
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --target $(cat rust_target.txt) --profile minimal --default-toolchain none
-ENV PATH="$HOME/.cargo/bin:$PATH"
-# Installs the correct toolchain version from rust-toolchain.toml and then the musl target
-RUN rustup target add $(cat rust_target.txt)
+RUN \
+  --mount=type=cache,target=/buildkit-cache,id="tool-caches" \
+  ( \
+    rustup self update \
+    || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --target $(cat rust_target.txt) --profile minimal --default-toolchain none \
+  ) \
+  # Installs the correct toolchain version from rust-toolchain.toml and then the musl target
+  && rustup target add $(cat rust_target.txt)
 
 # Build
-COPY crates crates
-COPY ./Cargo.toml Cargo.toml
-COPY ./Cargo.lock Cargo.lock
-RUN case "${TARGETPLATFORM}" in \
+RUN \
+  # bind mounts to access Cargo config, lock, and sources, without having to
+  # copy them into the build layer and so bloat the docker build cache
+  --mount=type=bind,source=crates,target=crates \
+  --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
+  --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
+  # Cache mounts to speed up builds
+  --mount=type=cache,target=$HOME/target/ \
+  --mount=type=cache,target=/buildkit-cache,id="tool-caches" \
+  case "${TARGETPLATFORM}" in \
   "linux/arm64") export JEMALLOC_SYS_WITH_LG_PAGE=16;; \
   esac && \
-  cargo zigbuild --bin uv --bin uvx --target $(cat rust_target.txt) --release
-RUN cp target/$(cat rust_target.txt)/release/uv /uv \
+  cargo zigbuild --bin uv --bin uvx --target $(cat rust_target.txt) --release \
+  && cp target/$(cat rust_target.txt)/release/uv /uv \
   && cp target/$(cat rust_target.txt)/release/uvx /uvx
 # TODO(konsti): Optimize binary size, with a version that also works when cross compiling
 # RUN strip --strip-all /uv