From 915cec327d3ff73d80ead162917b5d29382dfeeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Fri, 9 Aug 2024 19:12:20 +0200 Subject: [PATCH] Add nightly release (#2739) --- .github/workflows/assets.yml | 57 +++++++++ .github/workflows/deploy.yml | 124 ------------------- .github/workflows/release.yml | 95 ++++++++++++++ .github/workflows/test.yml | 22 ++-- .github/workflows/uffizzi-build.yml | 17 +-- .github/workflows/uffizzi-preview.yml | 2 +- Dockerfile | 96 ++++++++------ README.md | 2 +- docker/base/elixir-cuda.dockerfile | 21 ---- docker/build_and_push.sh | 69 ----------- lib/livebook/config.ex | 16 +-- test/livebook/hubs/dockerfile_test.exs | 22 ++-- test/livebook/runtime/fly_test.exs | 2 +- test/livebook_web/live/session_live_test.exs | 4 +- versions | 1 - 15 files changed, 250 insertions(+), 300 deletions(-) create mode 100644 .github/workflows/assets.yml delete mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/release.yml delete mode 100644 docker/base/elixir-cuda.dockerfile delete mode 100755 docker/build_and_push.sh diff --git a/.github/workflows/assets.yml b/.github/workflows/assets.yml new file mode 100644 index 000000000..266afc479 --- /dev/null +++ b/.github/workflows/assets.yml @@ -0,0 +1,57 @@ +name: Assets +on: + push: + branches: + - main + - "v*.*" +jobs: + build: + outputs: + sha: ${{ steps.push_assets.outputs.commit_hash }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + . versions + echo "elixir=$elixir" >> $GITHUB_ENV + echo "otp=$otp" >> $GITHUB_ENV + - name: Install Erlang & Elixir + uses: erlef/setup-beam@v1 + with: + otp-version: ${{ env.otp }} + elixir-version: ${{ env.elixir }} + - name: Cache Mix + uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}- + # Note: we need to get Phoenix and LV because package.json points to them directly + - name: Install mix dependencies + run: mix deps.get + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: "20.x" + - name: Cache npm dependencies + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: Clean generated assets + run: rm -rf static/assets + - name: Install npm dependencies + run: npm ci --prefix assets + - name: Build assets + run: npm run deploy --prefix assets + - name: Push updated assets + id: push_assets + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Update assets + file_pattern: static diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 0624bbbed..000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Deploy -on: - push: - branches: - - main - - "v*.*" - tags: - - "v*.*.*" -jobs: - assets: - outputs: - sha: ${{ steps.push_assets.outputs.commit_hash }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - run: | - . versions - echo "elixir=$elixir" >> $GITHUB_ENV - echo "otp=$otp" >> $GITHUB_ENV - - name: Install Erlang & Elixir - uses: erlef/setup-beam@v1 - with: - otp-version: ${{ env.otp }} - elixir-version: ${{ env.elixir }} - - name: Cache Mix - uses: actions/cache@v3 - with: - path: | - deps - _build - key: ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }} - restore-keys: | - ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}- - # Note: we need to get Phoenix and LV because package.json points to them directly - - name: Install mix dependencies - run: mix deps.get - - name: Install Node - uses: actions/setup-node@v3 - with: - node-version: "18.x" - - name: Cache npm dependencies - uses: actions/cache@v3 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - name: Clean generated assets - run: rm -rf static/assets - - name: Install npm dependencies - run: npm ci --prefix assets - - name: Build assets - run: npm run deploy --prefix assets - - name: Push updated assets - id: push_assets - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Update assets - file_pattern: static - docker: - if: github.ref_type == 'tag' || github.ref == 'refs/heads/main' - # The assets job may push new commit, so we wait for it - needs: [assets] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - # The output is set only if there was a commit, otherwise - # we pass an empty ref and and the default is used - ref: ${{ needs.assets.outputs.sha }} - - run: | - . versions - echo "elixir=$elixir" >> $GITHUB_ENV - echo "otp=$otp" >> $GITHUB_ENV - echo "debian=$debian" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta - id: meta - uses: docker/metadata-action@v3 - with: - images: ghcr.io/livebook-dev/livebook - tags: | - type=semver,pattern={{version}} - type=edge,branch=main - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - BASE_IMAGE=hexpm/elixir:${{ env.elixir }}-erlang-${{ env.otp }}-debian-${{ env.debian }} - - create_draft_release: - if: github.ref_type == 'tag' - permissions: - contents: write - runs-on: ubuntu-20.04 - env: - GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} - steps: - - name: Create draft release - run: | - gh release create \ - --repo ${{ github.repository }} \ - --title ${{ github.ref_name }} \ - --draft \ - ${{ github.ref_name }} - - - name: Trigger desktop builds - run: | - gh workflow run -R livebook-dev/livebook_cd build.yml -f ref_name=${{ github.ref_name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..3dbd7735f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,95 @@ +name: Release +on: + push: + tags: + - "v*.*.*" + schedule: + # Daily at 00:00 + - cron: "0 0 * * *" + # Workflow dispatch always builds as nightly + workflow_dispatch: +jobs: + create_draft_release: + if: github.ref_type == 'tag' + name: "Create draft release" + permissions: + contents: write + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} + steps: + - name: Create draft release + run: | + gh release create \ + --repo ${{ github.repository }} \ + --title ${{ github.ref_name }} \ + --draft \ + ${{ github.ref_name }} + + docker: + name: Docker (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: "default" + suffix: "" + build_args: | + VARIANT=default + - name: "cuda12" + tag_suffix: "-cuda12" + build_args: | + VARIANT=cuda + CUDA_VERSION=12-1 + steps: + - uses: actions/checkout@v4 + - run: | + . versions + echo "elixir=$elixir" >> $GITHUB_ENV + echo "otp=$otp" >> $GITHUB_ENV + echo "ubuntu=$ubuntu" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/livebook-dev/livebook + flavor: | + suffix=${{ matrix.tag_suffix }} + tags: | + type=semver,pattern={{version}} + type=raw,value=nightly,enable=${{ github.ref_type != 'tag' }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BASE_IMAGE=hexpm/elixir:${{ env.elixir }}-erlang-${{ env.otp }}-ubuntu-${{ env.ubuntu }} + ${{ matrix.build_args }} + + desktop: + name: "Desktop" + runs-on: ubuntu-latest + steps: + - name: Trigger desktop builds + run: | + if [[ "${{ github.ref_type }}" == "tag" ]]; then + gh workflow run -R livebook-dev/livebook_cd build.yml -f ref_name=${{ github.ref_name }} + else + gh workflow run -R livebook-dev/livebook_cd build.yml -f ref_name=${{ github.sha }} -f release_name=nightly + fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ad4abc9a..3a603fa50 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: ELIXIR_ERL_OPTIONS: "-epmd_module Elixir.Livebook.EPMD" steps: - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Read ./versions run: | . versions @@ -26,7 +26,7 @@ jobs: otp-version: ${{ env.otp }} elixir-version: ${{ env.elixir }} - name: Cache Mix - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | deps @@ -43,11 +43,11 @@ jobs: - name: Run tests run: mix test - name: Install Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: "18.x" + node-version: "20.x" - name: Cache npm dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -70,7 +70,7 @@ jobs: - name: Configure Git run: git config --global core.autocrlf input - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Read ./versions shell: bash run: | @@ -92,7 +92,7 @@ jobs: run: | "C:\Program Files\Git\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 - name: Cache Mix - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | deps @@ -113,7 +113,7 @@ jobs: if: github.event_name == 'push' steps: - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Read ./versions run: | . versions @@ -121,17 +121,17 @@ jobs: echo "otp=$otp" >> $GITHUB_ENV echo "openssl=$openssl" >> $GITHUB_ENV - name: Cache Mix Deps - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} - name: Cache Bootstrap OTP - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: elixirkit/otp_bootstrap/_build/otp-rel-${{ env.otp }}-openssl-${{ env.openssl }}-macos-universal key: ${{ runner.os }}-bootstrap-${{ env.otp }}-${{ hashFiles('elixirkit/otp_bootstrap/build.sh') }} - name: Cache Elixir - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: _build/elixir-${{ env.elixir }} key: ${{ runner.os }}-elixir-${{ env.elixir }} diff --git a/.github/workflows/uffizzi-build.yml b/.github/workflows/uffizzi-build.yml index 944bd95e8..630f246ed 100644 --- a/.github/workflows/uffizzi-build.yml +++ b/.github/workflows/uffizzi-build.yml @@ -12,12 +12,12 @@ jobs: tags: ${{ steps.meta.outputs.tags }} steps: - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - run: | . versions echo "elixir=$elixir" >> $GITHUB_ENV echo "otp=$otp" >> $GITHUB_ENV - echo "debian=$debian" >> $GITHUB_ENV + echo "ubuntu=$ubuntu" >> $GITHUB_ENV # --- START build assets - name: Install Erlang & Elixir uses: erlef/setup-beam@v1 @@ -25,7 +25,7 @@ jobs: otp-version: ${{ env.otp }} elixir-version: ${{ env.elixir }} - name: Cache Mix - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | deps @@ -37,11 +37,11 @@ jobs: - name: Install mix dependencies run: mix deps.get - name: Install Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: "18.x" + node-version: "20.x" - name: Cache npm dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -76,7 +76,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | - BASE_IMAGE=hexpm/elixir:${{ env.elixir }}-erlang-${{ env.otp }}-debian-${{ env.debian }} + BASE_IMAGE=hexpm/elixir:${{ env.elixir }}-erlang-${{ env.otp }}-ubuntu-${{ env.ubuntu }} + VARIANT=default render-compose-file: name: Render Docker Compose file @@ -86,7 +87,7 @@ jobs: - build-application steps: - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Render Compose File run: | APP_IMAGE=$(echo ${{ needs.build-application.outputs.tags }}) diff --git a/.github/workflows/uffizzi-preview.yml b/.github/workflows/uffizzi-preview.yml index f8e833e67..81e8794b2 100644 --- a/.github/workflows/uffizzi-preview.yml +++ b/.github/workflows/uffizzi-preview.yml @@ -59,7 +59,7 @@ jobs: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_OUTPUT - name: Cache Rendered Compose File if: ${{ steps.event.outputs.ACTION != 'closed' }} - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: docker-compose.rendered.yml key: ${{ steps.hash.outputs.COMPOSE_FILE_HASH }} diff --git a/Dockerfile b/Dockerfile index 40366c496..8b2f85474 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,34 @@ ARG BASE_IMAGE +ARG VARIANT -# Stage 1 -# Builds the Livebook release -FROM ${BASE_IMAGE} AS build +# Pre-stages for base image variants + +FROM ${BASE_IMAGE} as base-default + +FROM ${BASE_IMAGE} as base-cuda + +ARG CUDA_VERSION + +RUN distro="ubuntu$(. /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d '.')" && \ + # Official Docker images use the sbsa packages when targetting arm64. + # See https://gitlab.com/nvidia/container-images/cuda/-/blob/85f465ea3343a2d7f7753a0a838701999ed58a01/dist/12.5.1/ubuntu2204/base/Dockerfile#L12 + arch="$(if [ "$(uname -m)" = "aarch64" ]; then echo "sbsa"; else echo "x86_64"; fi)" && \ + apt update -q && apt install -y ca-certificates wget && \ + wget -qO /tmp/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.1-1_all.deb && \ + dpkg -i /tmp/cuda-keyring.deb && apt update -q + +# In order to minimize the image size, we install only a subset of +# the CUDA toolkit that is required by Elixir numerical packages +# (nvcc and runtime libraries). Note that we do not need to install +# the driver, it is already provided by NVIDIA Container Toolkit. +RUN apt install -y git cuda-nvcc-${CUDA_VERSION} cuda-libraries-${CUDA_VERSION} libcudnn8 + +# Build stage: builds the release +FROM base-${VARIANT} AS build RUN apt-get update && apt-get upgrade -y && \ - apt-get install --no-install-recommends -y \ - build-essential git && \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* + apt-get install --no-install-recommends -y \ + build-essential git WORKDIR /app @@ -39,49 +58,52 @@ COPY static static COPY iframe/priv/static/iframe iframe/priv/static/iframe COPY proto proto COPY lib lib -# We need README.md during compilation -# (look for @external_resource "README.md") +# We need README.md during compilation (look for @external_resource "README.md") COPY README.md README.md RUN mix do compile, release livebook -# Stage 2 -# Prepares the runtime environment and copies over the release. -# We use the same base image, because we need Erlang, Elixir and Mix -# during runtime to spawn the Livebook standalone runtimes. -# Consequently the release doesn't include ERTS as we have it anyway. -FROM ${BASE_IMAGE} +# Final stage: prepares the runtime environment and copies over the release. +# +# As opposed to usual deployments with Elixir releases, we want Erlang, +# Elixir and Mix to be available in the runtime image, because those +# are required for the Livebook runtimes. Consequently, we build the +# release with `include_erts: false`. +FROM base-${VARIANT} ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get upgrade -y && \ - apt-get install --no-install-recommends -y \ - # Runtime dependencies - build-essential ca-certificates libncurses5-dev \ - # In case someone uses `Mix.install/2` and point to a git repo - git \ - # Additional standard tools - wget \ - # In case someone uses Torchx for Nx - cmake && \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* + apt-get install --no-install-recommends -y \ + # Runtime dependencies + build-essential ca-certificates libncurses5-dev \ + # In case someone uses `Mix.install/2` and point to a git repo + git \ + # Additional standard tools + wget \ + # In case someone uses Torchx for Nx + cmake && \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \ + apt-get clean -y && \ + rm -rf /var/lib/apt/lists/* -# Run in the /data directory by default, makes for -# a good place for the user to mount local volume +# Run in the /data directory by default, makes for a good place for +# the user to mount local volume WORKDIR /data ENV HOME=/home/livebook -# Make sure someone running the container with `--user` -# has permissions to the home dir (for `Mix.install/2` cache) +# Make sure someone running the container with `--user` has permissions +# to the home dir (for `Mix.install/2` cache) RUN mkdir $HOME && chmod 777 $HOME # Install hex and rebar for `Mix.install/2` and Mix runtime RUN mix local.hex --force && \ - mix local.rebar --force + mix local.rebar --force -# Override the default 127.0.0.1 address, so that the app -# can be accessed outside the container by binding ports +# By default Livebook binds to loopback, but in order to make the app +# accessible outside of the container (by binding ports), we need to +# bind to any address. Also note that we specify IPv6 address, becuase +# we want to accept IPv6 connections. This is actually the default in +# usual Phoenix deployments. ENV LIVEBOOK_IP="::" ENV LIVEBOOK_HOME=/data @@ -89,8 +111,8 @@ ENV LIVEBOOK_HOME=/data # Copy the release build from the previous stage COPY --from=build /app/_build/prod/rel/livebook /app -# Make release files available to any user, in case someone -# runs the container with `--user` +# Make release files available to any user, in case someone runs the +# container with `--user` RUN chmod -R go=u /app # Make all home files available (specifically .mix/) RUN chmod -R go=u $HOME diff --git a/README.md b/README.md index 82fb88137..935a086e5 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ To deploy Livebook on your cloud platform, see our [Docker Deployment](docs/depl For CUDA support, [see images with the "cuda" tag](https://github.com/livebook-dev/livebook/pkgs/container/livebook). To try out features from the main branch you can alternatively -use the `ghcr.io/livebook-dev/livebook:edge` image. +use the `ghcr.io/livebook-dev/livebook:nightly` image. See [Livebook images](https://github.com/livebook-dev/livebook/pkgs/container/livebook). ### Embedded devices diff --git a/docker/base/elixir-cuda.dockerfile b/docker/base/elixir-cuda.dockerfile deleted file mode 100644 index ac7c955b1..000000000 --- a/docker/base/elixir-cuda.dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -ARG ELIXIR_VERSION -ARG ERLANG_VERSION -ARG UBUNTU_VERSION - -FROM hexpm/elixir:${ELIXIR_VERSION}-erlang-${ERLANG_VERSION}-ubuntu-${UBUNTU_VERSION} - -ARG CUDA_VERSION - -RUN distro="ubuntu$(. /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d '.')" && \ - # Official Docker images use the sbsa packages when targetting arm64. - # See https://gitlab.com/nvidia/container-images/cuda/-/blob/85f465ea3343a2d7f7753a0a838701999ed58a01/dist/12.5.1/ubuntu2204/base/Dockerfile#L12 - arch="$(if [ "$(uname -m)" = "aarch64" ]; then echo "sbsa"; else echo "x86_64"; fi)" && \ - apt update -q && apt install -y ca-certificates wget && \ - wget -qO /tmp/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-keyring_1.1-1_all.deb && \ - dpkg -i /tmp/cuda-keyring.deb && apt update -q - -# In order to minimize the image size, we install only a subset of -# the CUDA toolkit that is required by Elixir numerical packages -# (nvcc and runtime libraries). Note that we do not need to install -# the driver, it is already provided by NVIDIA Container Toolkit. -RUN apt install -y git cuda-nvcc-${CUDA_VERSION} cuda-libraries-${CUDA_VERSION} libcudnn8 diff --git a/docker/build_and_push.sh b/docker/build_and_push.sh deleted file mode 100755 index 20f042f6b..000000000 --- a/docker/build_and_push.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Builds extra Livebook images. -# -# Images are pushed to GHCR, so you need to make sure that the docker -# client is authenticated. - -set -ex -cd "$(dirname "$0")/.." - -. versions - -livebook_version="$(cat mix.exs | grep '@version "' | grep -o '".*"' | tr -d '"')" - -if [ -z "$livebook_version" ] || [[ "$livebook_version" == *"dev"* ]]; then - echo "No releasable Livebook version detected" - exit 1 -fi - -if [[ "$livebook_version" == *"-"* ]]; then - stable=false -else - stable=true -fi - -cuda_tag_list=("cuda11.8" "cuda12.1") -cuda_version_list=("11-8" "12-1") - -for idx in "${!cuda_tag_list[@]}"; do - cuda_tag="${cuda_tag_list[idx]}" - cuda_version="${cuda_version_list[idx]}" - - base_image="ghcr.io/livebook-dev/utils:elixir-$elixir-erlang-$otp-$cuda_tag" - - if docker manifest inspect $base_image > /dev/null; then - echo "Using base image: $base_image" - else - echo "Building base image: $base_image" - - docker buildx build --push --platform linux/amd64,linux/arm64 \ - -t $base_image \ - --build-arg ELIXIR_VERSION=$elixir \ - --build-arg ERLANG_VERSION=$otp \ - --build-arg UBUNTU_VERSION=$ubuntu \ - --build-arg CUDA_VERSION=$cuda_version \ - -f docker/base/elixir-cuda.dockerfile \ - docker/base - fi - - image="ghcr.io/livebook-dev/livebook:$livebook_version-$cuda_tag" - - if docker manifest inspect $image > /dev/null; then - echo "Skipping image, since it already exists: $image" - else - echo "Building image: $image" - - extra_args="" - - if [[ "$stable" == "true" ]]; then - extra_args="-t ghcr.io/livebook-dev/livebook:latest-$cuda_tag" - fi - - docker buildx build --push --platform linux/amd64,linux/arm64 \ - --build-arg BASE_IMAGE="$base_image" \ - -t $image \ - $extra_args \ - . - fi -done diff --git a/lib/livebook/config.ex b/lib/livebook/config.ex index 70534294d..69b73d944 100644 --- a/lib/livebook/config.ex +++ b/lib/livebook/config.ex @@ -61,23 +61,13 @@ defmodule Livebook.Config do def docker_images() do version = app_version() - {version, version_cuda} = - if version =~ "dev" do - {"edge", "latest"} - else - {version, version} - end + version = if version =~ "dev", do: "nightly", else: version [ %{tag: version, name: "Livebook", env: []}, %{ - tag: "#{version_cuda}-cuda11.8", - name: "Livebook + CUDA 11.8", - env: [{"XLA_TARGET", "cuda118"}] - }, - %{ - tag: "#{version_cuda}-cuda12.1", - name: "Livebook + CUDA 12.1", + tag: "#{version}-cuda12", + name: "Livebook + CUDA 12", env: [{"XLA_TARGET", "cuda120"}] } ] diff --git a/test/livebook/hubs/dockerfile_test.exs b/test/livebook/hubs/dockerfile_test.exs index 559016f22..18bd82d51 100644 --- a/test/livebook/hubs/dockerfile_test.exs +++ b/test/livebook/hubs/dockerfile_test.exs @@ -7,9 +7,9 @@ defmodule Livebook.Hubs.DockerfileTest do alias Livebook.Hubs alias Livebook.Secrets.Secret - @versions if Livebook.Config.app_version() =~ "-dev", - do: %{base: "edge", cuda: "latest"}, - else: %{base: Livebook.Config.app_version(), cuda: Livebook.Config.app_version()} + @version if Livebook.Config.app_version() =~ "-dev", + do: "nightly", + else: Livebook.Config.app_version() describe "airgapped_dockerfile/7" do test "deploying a single notebook in personal hub" do @@ -20,7 +20,7 @@ defmodule Livebook.Hubs.DockerfileTest do dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile == """ - FROM ghcr.io/livebook-dev/livebook:#{@versions.base} + FROM ghcr.io/livebook-dev/livebook:#{@version} # Apps configuration ENV LIVEBOOK_APPS_PATH "/apps" @@ -97,7 +97,7 @@ defmodule Livebook.Hubs.DockerfileTest do dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile == """ - FROM ghcr.io/livebook-dev/livebook:#{@versions.base} + FROM ghcr.io/livebook-dev/livebook:#{@version} ARG TEAMS_KEY="lb_tk_fn0pL3YLWzPoPFWuHeV3kd0o7_SFuIOoU4C_k6OWDYg" @@ -166,16 +166,16 @@ defmodule Livebook.Hubs.DockerfileTest do end test "deploying with different base image" do - config = dockerfile_config(%{docker_tag: "#{@versions.cuda}-cuda11.8"}) + config = dockerfile_config(%{docker_tag: "#{@version}-cuda12"}) hub = personal_hub() file = Livebook.FileSystem.File.local(p("/notebook.livemd")) dockerfile = Dockerfile.airgapped_dockerfile(config, hub, [], [], file, [], %{}) assert dockerfile =~ """ - FROM ghcr.io/livebook-dev/livebook:#{@versions.cuda}-cuda11.8 + FROM ghcr.io/livebook-dev/livebook:#{@version}-cuda12 - ENV XLA_TARGET "cuda118" + ENV XLA_TARGET "cuda120" """ end @@ -247,14 +247,14 @@ defmodule Livebook.Hubs.DockerfileTest do end test "deploying with different base image" do - config = dockerfile_config(%{docker_tag: "#{@versions.cuda}-cuda11.8"}) + config = dockerfile_config(%{docker_tag: "#{@version}-cuda12"}) hub = team_hub() agent_key = Livebook.Factory.build(:agent_key) %{image: image, env: env} = Dockerfile.online_docker_info(config, hub, agent_key) - assert image == "ghcr.io/livebook-dev/livebook:#{@versions.cuda}-cuda11.8" - assert {"XLA_TARGET", "cuda118"} in env + assert image == "ghcr.io/livebook-dev/livebook:#{@version}-cuda12" + assert {"XLA_TARGET", "cuda120"} in env end test "deploying with auto cluster setup" do diff --git a/test/livebook/runtime/fly_test.exs b/test/livebook/runtime/fly_test.exs index 207e3a6ca..c07f8670a 100644 --- a/test/livebook/runtime/fly_test.exs +++ b/test/livebook/runtime/fly_test.exs @@ -73,7 +73,7 @@ defmodule Livebook.Runtime.FlyTest do gpu_kind: nil, gpus: nil, volume_id: nil, - docker_tag: "edge" + docker_tag: "nightly" } Map.merge(defaults, attrs) diff --git a/test/livebook_web/live/session_live_test.exs b/test/livebook_web/live/session_live_test.exs index 780f26b1c..b5ce94ecc 100644 --- a/test/livebook_web/live/session_live_test.exs +++ b/test/livebook_web/live/session_live_test.exs @@ -1111,7 +1111,7 @@ defmodule LivebookWeb.SessionLiveTest do gpu_kind: nil, gpus: nil, volume_id: "vol_1", - docker_tag: "edge" + docker_tag: "nightly" }) Session.set_runtime(session.pid, runtime) @@ -1183,7 +1183,7 @@ defmodule LivebookWeb.SessionLiveTest do gpu_kind: nil, gpus: nil, volume_id: "vol_1", - docker_tag: "edge" + docker_tag: "nightly" }) Session.set_runtime(session.pid, runtime) diff --git a/versions b/versions index 8c2bbb299..7d18252ae 100644 --- a/versions +++ b/versions @@ -2,5 +2,4 @@ elixir="1.17.2" otp="27.0" openssl="1.1.1s" rebar3="3.22.0" -debian="bookworm-20240701-slim" ubuntu="jammy-20240530"