diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 057ec2801..7f81817a2 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -92,6 +92,23 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+  docker_extras:
+    if: github.ref_type == 'tag'
+    # The assets job may push new commit, so we wait for it
+    needs: [assets]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        run: |
+          version=${GITHUB_REF#refs/*/v}
+          docker/build_and_push.sh $version,latest true
+
   create_draft_release:
     if: github.ref_type == 'tag'
     permissions:
diff --git a/Dockerfile b/Dockerfile
index afba26431..15e8e0a1d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,8 @@
+ARG BASE_IMAGE=hexpm/elixir:1.14.2-erlang-24.3.4.2-debian-bullseye-20210902-slim
+
 # Stage 1
 # Builds the Livebook release
-FROM hexpm/elixir:1.14.2-erlang-24.3.4.2-debian-bullseye-20210902-slim AS build
+FROM ${BASE_IMAGE} AS build
 
 RUN apt-get update && apt-get upgrade -y && \
     apt-get install --no-install-recommends -y \
@@ -39,7 +41,7 @@ RUN mix do compile, release livebook
 # We use the same base image, because we need Erlang, Elixir and Mix
 # during runtime to spawn the Livebook standalone runtimes.
 # Consequently the release doesn't include ERTS as we have it anyway.
-FROM hexpm/elixir:1.14.2-erlang-24.3.4.2-debian-bullseye-20210902-slim
+FROM ${BASE_IMAGE}
 
 RUN apt-get update && apt-get upgrade -y && \
     apt-get install --no-install-recommends -y \
diff --git a/docker/base/elixir-cuda.dockerfile b/docker/base/elixir-cuda.dockerfile
new file mode 100644
index 000000000..ab29b5a8f
--- /dev/null
+++ b/docker/base/elixir-cuda.dockerfile
@@ -0,0 +1,25 @@
+ARG ELIXIR_VERSION
+ARG ERLANG_VERSION
+
+ARG CUDA_VERSION
+
+FROM hexpm/elixir:${ELIXIR_VERSION}-erlang-${ERLANG_VERSION}-ubuntu-focal-20211006 AS elixir
+
+FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-devel-ubuntu20.04
+
+ENV DEBIAN_FRONTEND nonintaeractive
+ENV LANG=C.UTF-8
+
+# Erlang runtime dependencies, see https://github.com/hexpm/bob/blob/3b5721dccdfe9d59766f374e7b4fb7fb8a7c720e/priv/scripts/docker/erlang-ubuntu-focal.dockerfile#L41-L45
+RUN apt-get update && \
+  apt-get -y --no-install-recommends install \
+    libodbc1 \
+    libssl1.1 \
+    libsctp1
+
+# We copy the top-level directory first to preserve symlinks in /usr/local/bin
+COPY --from=elixir /usr/local /usr/ELIXIR_LOCAL
+
+RUN cp -r /usr/ELIXIR_LOCAL/lib/* /usr/local/lib && \
+  cp -r /usr/ELIXIR_LOCAL/bin/* /usr/local/bin && \
+  rm -rf /usr/ELIXIR_LOCAL
diff --git a/docker/build_and_push.sh b/docker/build_and_push.sh
new file mode 100755
index 000000000..7029b13df
--- /dev/null
+++ b/docker/build_and_push.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+cd "$(dirname "$0")/.."
+
+print_usage_and_exit() {
+  echo "Usage: $0 <versions> <push>"
+  echo ""
+  echo "Builds extra variants of the regular Livebook Docker image."
+  echo ""
+  echo "<versions> is a comma-separated list of versions to use in the image tags (such as 0.1.0 or latest)."
+  echo "Set <push> to 'true' to also pushe the images."
+  exit 1
+}
+
+if [ $# -ne 2 ]; then
+  print_usage_and_exit
+fi
+
+versions=$1
+push=$2
+
+elixir=1.14.2
+erlang=24.3.4.2
+
+docker build -t elixir-cuda11.8 --build-arg ELIXIR_VERSION=$elixir --build-arg ERLANG_VERSION=$erlang --build-arg CUDA_VERSION=11.8.0 -f docker/base/elixir-cuda.dockerfile docker/base
+docker build -t livebook-cuda11.8 --build-arg BASE_IMAGE=elixir-cuda11.8 .
+
+for version in ${versions//,/ }; do
+  image=livebook/livebook:$version-cuda11.8
+  docker tag livebook-cuda11.8 $image
+
+  if [[ "$2" == "true" ]]; then
+    docker push $image
+  fi
+done