Simplify static assets handling (#2866)

This commit is contained in:
Jonatan Kłosko 2024-11-21 11:46:33 +01:00 committed by GitHub
parent 96a995bf26
commit 21e01fadb6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 164 additions and 423 deletions

View file

@ -12,5 +12,7 @@ npm-debug.log
/assets/node_modules/
/tmp/
/livebook
/priv/static
/priv/iframe_static
# Ignore app release files, including build artifacts
/rel/app

4
.gitignore vendored
View file

@ -33,3 +33,7 @@ npm-debug.log
# The built Escript
/livebook
# We generate priv when building release or escript
/priv/static
/priv/iframe_static

View file

@ -57,6 +57,7 @@ COPY config config
RUN mix do deps.get, deps.compile
# Compile and build the release
COPY priv/.gitkeep priv/.gitkeep
COPY rel rel
COPY static static
COPY iframe/priv/static/iframe iframe/priv/static/iframe

View file

@ -226,7 +226,7 @@ export function setFavicon(name) {
document.head.appendChild(link);
}
link.href = `/${name}.svg`;
link.href = `/favicons/${name}.svg`;
}
export function findChildOrThrow(element, selector) {

View file

@ -8,6 +8,18 @@ defmodule Livebook.Config do
| %{mode: :token, secret: String.t()}
| %{mode: :disabled}
@doc """
Returns path to Livebook priv directory.
This returns the usual priv directory, however in case of Escript,
the priv files are extracted into a temporary directory and that is
the path returned.
"""
@spec priv_path() :: String.t()
def priv_path() do
Application.get_env(:livebook, :priv_dir) || Application.app_dir(:livebook, "priv")
end
@doc """
Returns docker images to be used when generating sample Dockerfiles.
"""

View file

@ -13,6 +13,9 @@ defmodule LivebookCLI do
def main(args) do
{:ok, _} = Application.ensure_all_started(:elixir)
extract_priv!()
:ok = Application.load(:livebook)
if unix?() do
@ -76,4 +79,51 @@ defmodule LivebookCLI do
version = Livebook.Config.app_version()
IO.puts("\nLivebook #{version}")
end
import Record
defrecord(:zip_file, extract(:zip_file, from_lib: "stdlib/include/zip.hrl"))
defp extract_priv!() do
archive_dir = Path.join(Livebook.Config.tmp_path(), "escript")
extracted_path = Path.join(archive_dir, ".extracted")
in_archive_priv_path = ~c"livebook/priv"
# In dev we want to extract fresh directory on every boot
if Livebook.Config.app_version() =~ "-dev" do
File.rm_rf!(archive_dir)
end
# When temporary directory is cleaned by the OS, the directories
# may be left in place, so we use a regular file (.extracted) to
# check if the extracted archive is already available
if not File.exists?(extracted_path) do
{:ok, sections} = :escript.extract(:escript.script_name(), [])
archive = Keyword.fetch!(sections, :archive)
file_filter = fn zip_file(name: name) ->
List.starts_with?(name, in_archive_priv_path)
end
case :zip.extract(archive, cwd: String.to_charlist(archive_dir), file_filter: file_filter) do
{:ok, _} ->
:ok
{:error, error} ->
print_error_and_exit(
"Livebook failed to extract archive files, reason: #{inspect(error)}"
)
end
File.touch!(extracted_path)
end
priv_dir = Path.join(archive_dir, in_archive_priv_path)
Application.put_env(:livebook, :priv_dir, priv_dir, persistent: true)
end
@spec print_error_and_exit(String.t()) :: no_return()
defp print_error_and_exit(message) do
IO.ANSI.format([:red, message]) |> IO.puts()
System.halt(1)
end
end

View file

@ -1,5 +1,5 @@
defmodule LivebookWeb do
def static_paths, do: ~w(assets images favicon.svg favicon.png robots.txt)
def static_paths, do: ~w(assets images favicons robots.txt)
def controller do
quote do

View file

@ -5,8 +5,8 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="csrf-token" content={get_csrf_token()} />
<link rel="icon" type="image/svg+xml" href={~p"/favicon.svg"} />
<link rel="alternate icon" type="image/png" href={~p"/favicon.png"} />
<link rel="icon" type="image/svg+xml" href={~p"/favicons/favicon.svg"} />
<link rel="alternate icon" type="image/png" href={~p"/favicons/favicon.png"} />
<.live_title>
<%= assigns[:page_title] || "Livebook" %>
</.live_title>

View file

@ -41,8 +41,8 @@ defmodule LivebookWeb.ErrorHTML do
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" type="image/svg+xml" href={~p"/favicon.svg"} />
<link rel="alternate icon" type="image/png" href={~p"/favicon.png"} />
<link rel="icon" type="image/svg+xml" href={~p"/favicons/favicon.svg"} />
<link rel="alternate icon" type="image/png" href={~p"/favicons/favicon.png"} />
<title><%= @status %> - Livebook</title>
<link rel="stylesheet" href={~p"/assets/app.css"} />
</head>

View file

@ -21,37 +21,49 @@ defmodule LivebookWeb.Endpoint do
socket "/live", Phoenix.LiveView.Socket, websocket: @websocket_options
socket "/socket", LivebookWeb.Socket, websocket: @websocket_options
# We use Escript for distributing Livebook, so we don't have access to the static
# files at runtime in the prod environment. To overcome this we load contents of
# those files at compilation time, so that they become a part of the executable
# and can be served from memory.
defmodule AssetsMemoryProvider do
use LivebookWeb.MemoryProvider,
from: Path.expand("../../static", __DIR__),
gzip: true
end
defmodule AssetsFileSystemProvider do
use LivebookWeb.FileSystemProvider,
from: "tmp/static_dev"
end
# Serve static files at "/"
# Serve static files at "/".
#
# In usual Phoenix applications, we serve static files from priv/static,
# however Livebook can also be run as escript, in which case it is
# packaged into a single file and priv/ is not accessible directly.
# In that case, we include priv/ in the escript archive by setting
# the :include_priv_for option. Then, on escript boot, we extract
# the priv files into a temporary directory.
#
# To account for both cases, we configure Plug.Static :from as MFA
# and return the accessible priv/ location in both scenarios.
#
# The priv/ static files are generated by the livebook.gen_priv task
# before building the escript or the release. We gzip the static
# files in priv/, since we want to serve them gzipped, and we don't
# include the non-gzipped ones to minimize app size. Note that we
# still have a separate static/ directory with the CI-precompiled
# assets, which we keep in Git so that people can install escript
# from GitHub or run MIX_ENV=prod phx.server, without Node and NPM.
# Storing minified assets is already not ideal, but we definitely
# want to avoid storing the gzipped variants in Git. That's why we
# store the assets uncompressed and then generate priv/static with
# their compressed variants as part of the build process.
if code_reloading? do
# In development we use assets from tmp/static_dev (rebuilt dynamically on every change).
# Note that this directory doesn't contain predefined files (e.g. images), so we also
# use `AssetsMemoryProvider` to serve those from static/.
plug LivebookWeb.StaticPlug,
# In development, we use assets from tmp/static_dev, which are
# rebuilt on every change. We build to a different directory than
# priv/static, to make sure it can be built concurrently.
plug Plug.Static,
at: "/",
file_provider: AssetsFileSystemProvider,
gzip: false
from: "tmp/static_dev",
gzip: false,
only: ["assets"]
end
plug LivebookWeb.StaticPlug,
plug Plug.Static,
at: "/",
file_provider: AssetsMemoryProvider,
gzip: true
from: {__MODULE__, :static_from, []},
gzip: true,
only: LivebookWeb.static_paths()
@doc false
def static_from(), do: Path.join(Livebook.Config.priv_path(), "static")
plug :force_ssl

View file

@ -1,25 +1,20 @@
defmodule LivebookWeb.IframeEndpoint do
use Plug.Builder
defmodule AssetsMemoryProvider do
use LivebookWeb.MemoryProvider,
from: Path.expand("../../iframe/priv/static/iframe", __DIR__),
gzip: true
end
plug LivebookWeb.StaticPlug,
plug Plug.Static,
at: "/iframe",
file_provider: AssetsMemoryProvider,
gzip: true,
from: {__MODULE__, :static_from, []},
# Iframes are versioned, so we cache them for long
cache_control_for_etags: "public, max-age=31536000",
headers: [
# Enable CORS to allow Livebook fetch the content and verify its integrity
{"access-control-allow-origin", "*"},
# Iframes are versioned, so we cache them for long
{"cache-control", "public, max-age=31536000"},
# Specify the charset
{"content-type", "text/html; charset=utf-8"}
]
@doc false
def static_from(), do: Path.join(Livebook.Config.priv_path(), "static_iframe")
plug :not_found
defp not_found(conn, _) do

View file

@ -1,36 +0,0 @@
defmodule LivebookWeb.FileSystemProvider do
# Configurable implementation of `LivebookWeb.StaticPlug.Provider` behaviour,
# that loads files directly from the file system.
#
# ## `use` options
#
# * `:from` (**required**) - where to read the static files from. See `Plug.Static` for more details.
defmacro __using__(opts) do
quote bind_quoted: [opts: opts] do
@behaviour LivebookWeb.StaticPlug.Provider
from = Keyword.fetch!(opts, :from)
static_path = LivebookWeb.StaticPlug.Provider.static_path(from)
@impl true
def get_file(segments, compression) do
LivebookWeb.FileSystemProvider.__get_file__(unquote(static_path), segments, compression)
end
end
end
def __get_file__(static_path, segments, nil) do
abs_path = Path.join([static_path | segments])
if File.regular?(abs_path) do
content = File.read!(abs_path)
digest = content |> :erlang.md5() |> Base.encode16(case: :lower)
%LivebookWeb.StaticPlug.File{content: content, digest: digest}
else
nil
end
end
def __get_file__(_static_path, _segments, _compression), do: nil
end

View file

@ -1,71 +0,0 @@
defmodule LivebookWeb.MemoryProvider do
@gzippable_exts ~w(.js .css .txt .text .html .json .svg .eot .ttf)
# Configurable implementation of `LivebookWeb.StaticPlug.Provider` behaviour,
# that bundles the files into the module compiled source.
#
# ## `use` options
#
# * `:from` (**required**) - where to read the static files from. See `Plug.Static` for more details.
#
# * `:gzip` - whether to bundle gzipped version of the files,
# in which case the uncompressed files are not included. Defaults to `false`.
defmacro __using__(opts) do
quote bind_quoted: [opts: opts] do
@behaviour LivebookWeb.StaticPlug.Provider
from = Keyword.fetch!(opts, :from)
static_path = LivebookWeb.StaticPlug.Provider.static_path(from)
paths = LivebookWeb.MemoryProvider.__paths__(static_path)
files = LivebookWeb.MemoryProvider.__preload_files__!(static_path, paths, opts)
for path <- paths do
abs_path = Path.join(static_path, path)
@external_resource Path.relative_to_cwd(abs_path)
end
@impl true
def get_file(segments, compression)
for {segments, compression, file} <- files do
def get_file(unquote(segments), unquote(compression)), do: unquote(Macro.escape(file))
end
def get_file(_, _), do: nil
# Force recompilation if the static files change.
def __mix_recompile__? do
current_paths = LivebookWeb.MemoryProvider.__paths__(unquote(static_path))
:erlang.md5(current_paths) != unquote(:erlang.md5(paths))
end
end
end
def __preload_files__!(static_path, paths, opts) do
gzip? = Keyword.get(opts, :gzip, false)
Enum.map(paths, fn path ->
segments = Path.split(path)
abs_path = Path.join(static_path, path)
content = File.read!(abs_path)
digest = content |> :erlang.md5() |> Base.encode16(case: :lower)
if gzip? and Path.extname(path) in @gzippable_exts do
gzipped_content = :zlib.gzip(content)
{segments, :gzip, %LivebookWeb.StaticPlug.File{content: gzipped_content, digest: digest}}
else
{segments, nil, %LivebookWeb.StaticPlug.File{content: content, digest: digest}}
end
end)
end
def __paths__(static_path) do
Path.join(static_path, "**")
|> Path.wildcard()
|> Enum.reject(&File.dir?/1)
|> Enum.map(&String.replace_leading(&1, static_path <> "/", ""))
|> Enum.sort()
end
end

View file

@ -1,160 +0,0 @@
defmodule LivebookWeb.StaticPlug.File do
defstruct [:content, :digest]
@type t :: %__MODULE__{content: binary(), digest: String.t()}
end
defmodule LivebookWeb.StaticPlug.Provider do
@type segments :: list(String.t())
@type compression :: :gzip | nil
@doc """
Returns file data for the given path (given as list of segments) and compression type.
"""
@callback get_file(segments(), compression()) :: LivebookWeb.StaticPlug.File.t() | nil
@doc """
Parses static files location usually passed as the `:from` option
when configuring provider.
See `Plug.Static` for more details.
"""
@spec static_path({atom(), binary()} | atom() | binary()) :: binary()
def static_path(from)
def static_path({app, path}) when is_atom(app) and is_binary(path) do
Path.join(Application.app_dir(app), path)
end
def static_path(path) when is_binary(path), do: path
def static_path(app) when is_atom(app), do: static_path({app, "priv/static"})
end
defmodule LivebookWeb.StaticPlug do
# This is a simplified version of `Plug.Static` meant
# to serve static files using the given provider.
#
# ## Options
#
# * `:file_provider` (**required**) - a module implementing `LivebookWeb.StaticPlug.Provider`
# behaviour, responsible for resolving file requests
#
# * `:at`, `:gzip`, `:headers` - same as `Plug.Static`
@behaviour Plug
import Plug.Conn
@allowed_methods ~w(GET HEAD)
@impl true
def init(opts) do
file_provider = Keyword.fetch!(opts, :file_provider)
%{
file_provider: file_provider,
at: opts |> Keyword.fetch!(:at) |> Plug.Router.Utils.split(),
gzip?: Keyword.get(opts, :gzip, false),
headers: Keyword.get(opts, :headers, [])
}
end
@impl true
def call(
%Plug.Conn{method: method} = conn,
%{file_provider: file_provider, at: at, gzip?: gzip?} = options
)
when method in @allowed_methods do
segments = subset(at, conn.path_info)
case encoding_with_file(conn, file_provider, segments, gzip?) do
{encoding, file} ->
serve_static(conn, encoding, file, segments, options)
:error ->
conn
end
end
def call(conn, _options) do
conn
end
defp serve_static(conn, content_encoding, file, segments, options) do
case put_cache_header(conn, file) do
{:stale, conn} ->
filename = List.last(segments)
content_type = MIME.from_path(filename)
conn
|> put_resp_header("content-type", content_type)
|> maybe_add_encoding(content_encoding)
|> maybe_add_vary(options)
|> merge_resp_headers(options.headers)
|> send_resp(200, file.content)
|> halt()
{:fresh, conn} ->
conn
|> maybe_add_vary(options)
|> send_resp(304, "")
|> halt()
end
end
defp maybe_add_encoding(conn, nil), do: conn
defp maybe_add_encoding(conn, ce), do: put_resp_header(conn, "content-encoding", ce)
# If we serve gzip at any moment, we need to set the proper vary
# header regardless of whether we are serving gzip content right now.
# See: http://www.fastly.com/blog/best-practices-for-using-the-vary-header/
defp maybe_add_vary(conn, %{gzip?: true}) do
update_in(conn.resp_headers, &[{"vary", "Accept-Encoding"} | &1])
end
defp maybe_add_vary(conn, _options), do: conn
defp put_cache_header(conn, file) do
etag = etag_for_file(file)
conn =
conn
|> put_resp_header("cache-control", "public")
|> put_resp_header("etag", etag)
if etag in get_req_header(conn, "if-none-match") do
{:fresh, conn}
else
{:stale, conn}
end
end
defp etag_for_file(file) do
<<?", file.digest::binary, ?">>
end
defp encoding_with_file(conn, file_provider, segments, gzip?) do
cond do
file = gzip? and accept_encoding?(conn, "gzip") && file_provider.get_file(segments, :gzip) ->
{"gzip", file}
file = file_provider.get_file(segments, nil) ->
{nil, file}
true ->
:error
end
end
defp accept_encoding?(conn, encoding) do
encoding? = &String.contains?(&1, [encoding, "*"])
Enum.any?(get_req_header(conn, "accept-encoding"), fn accept ->
accept |> Plug.Conn.Utils.list() |> Enum.any?(encoding?)
end)
end
defp subset([h | expected], [h | actual]), do: subset(expected, actual)
defp subset([], actual), do: actual
defp subset(_, _), do: []
end

View file

@ -0,0 +1,37 @@
defmodule Mix.Tasks.Livebook.GenPriv do
@moduledoc false
# Note that we need to include priv/.gitkeep in Dockerfile and Hex
# package files, so that priv/ is symlinked within _build/, before
# we generate the actual files.
use Mix.Task
@gzippable_exts ~w(.js .css .txt .text .html .json .svg .eot .ttf)
@impl true
def run([]) do
compress_and_copy("static", "priv/static")
compress_and_copy("iframe/priv/static/iframe", "priv/iframe_static")
end
defp compress_and_copy(source_dir, target_dir) do
File.rm_rf!(target_dir)
source_paths = Path.wildcard(Path.join(source_dir, "**/*"))
for source_path <- source_paths, File.regular?(source_path) do
target_path = Path.join(target_dir, Path.relative_to(source_path, source_dir))
File.mkdir_p!(Path.dirname(target_path))
if Path.extname(source_path) in @gzippable_exts do
content = source_path |> File.read!() |> :zlib.gzip()
File.write!(target_path <> ".gz", content)
else
File.cp!(source_path, target_path)
end
end
Mix.shell().info("Generated #{target_dir} with compressed files from #{source_dir}")
end
end

10
mix.exs
View file

@ -59,7 +59,7 @@ defmodule Livebook.MixProject do
"GitHub" => "https://github.com/livebook-dev/livebook"
},
files:
~w(lib static config mix.exs mix.lock README.md LICENSE CHANGELOG.md iframe/priv/static/iframe proto/lib)
~w(lib static priv/.gitkeep config mix.exs mix.lock README.md LICENSE CHANGELOG.md iframe/priv/static/iframe proto/lib)
]
end
@ -68,7 +68,10 @@ defmodule Livebook.MixProject do
setup: ["deps.get", "cmd --cd assets npm install"],
"assets.deploy": ["cmd npm run deploy --prefix assets"],
"format.all": ["format", "cmd --cd assets npm run format"],
"protobuf.generate": ["cmd --cd proto mix protobuf.generate"]
"protobuf.generate": ["cmd --cd proto mix protobuf.generate"],
"phx.server": ["livebook.gen_priv", "phx.server"],
"escript.build": ["livebook.gen_priv", "escript.build"],
release: ["livebook.gen_priv", "release"]
]
end
@ -76,7 +79,8 @@ defmodule Livebook.MixProject do
[
main_module: LivebookCLI,
app: nil,
emu_args: "-epmd_module Elixir.Livebook.EPMD"
emu_args: "-epmd_module Elixir.Livebook.EPMD",
include_priv_for: [:livebook]
]
end

0
priv/.gitkeep Normal file
View file

File diff suppressed because one or more lines are too long

View file

Before

Width:  |  Height:  |  Size: 9.1 KiB

After

Width:  |  Height:  |  Size: 9.1 KiB

View file

Before

Width:  |  Height:  |  Size: 8.8 KiB

After

Width:  |  Height:  |  Size: 8.8 KiB

View file

Before

Width:  |  Height:  |  Size: 9.1 KiB

After

Width:  |  Height:  |  Size: 9.1 KiB

View file

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

View file

Before

Width:  |  Height:  |  Size: 7.9 KiB

After

Width:  |  Height:  |  Size: 7.9 KiB

View file

@ -1,21 +0,0 @@
defmodule LivebookWeb.FileSystemProviderTest do
use ExUnit.Case, async: true
defmodule MyProvider do
use LivebookWeb.FileSystemProvider,
from: Path.expand("../../support/static", __DIR__)
end
test "includes regular files" do
assert %{content: content} = MyProvider.get_file(["js", "app.js"], nil)
assert content =~ ~s{console.log("Hello");}
end
test "ignores directories" do
assert nil == MyProvider.get_file(["js"], nil)
end
test "ignores non-existent files" do
assert nil == MyProvider.get_file(["nonexistent.js"], nil)
end
end

View file

@ -1,22 +0,0 @@
defmodule LivebookWeb.MemoryProviderTest do
use ExUnit.Case, async: true
defmodule MyProvider do
use LivebookWeb.MemoryProvider,
from: Path.expand("../../support/static", __DIR__),
gzip: true
end
test "includes uncompressed files that are not gzippable" do
assert %{content: ""} = MyProvider.get_file(["icon.ico"], nil)
end
test "includes compressed files which are gzippable" do
assert %{content: content} = MyProvider.get_file(["js", "app.js"], :gzip)
assert :zlib.gunzip(content) =~ ~s{console.log("Hello");}
end
test "does not include uncompressed files that are gzippable" do
assert nil == MyProvider.get_file(["js", "app.js"], nil)
end
end

View file

@ -1,66 +0,0 @@
defmodule LivebookWeb.StaticPlugTest do
use ExUnit.Case, async: true
use Plug.Test
defmodule MyProvider do
@behaviour LivebookWeb.StaticPlug.Provider
@impl true
def get_file(["app.js"], :gzip) do
%LivebookWeb.StaticPlug.File{content: "content", digest: "digest"}
end
def get_file(["icon.ico"], nil) do
%LivebookWeb.StaticPlug.File{content: "content", digest: "digest"}
end
def get_file(_path, _compression), do: nil
end
defmodule MyPlug do
use Plug.Builder
plug LivebookWeb.StaticPlug,
at: "/",
file_provider: MyProvider,
gzip: true
plug :passthrough
defp passthrough(conn, _), do: Plug.Conn.send_resp(conn, 404, "Passthrough")
end
defp call(conn), do: MyPlug.call(conn, [])
test "serves uncompressed file if there is no compressed version" do
conn =
conn(:get, "/icon.ico")
|> put_req_header("accept-encoding", "gzip")
|> call()
assert conn.status == 200
assert conn.resp_body == "content"
assert get_resp_header(conn, "content-type") == ["image/vnd.microsoft.icon"]
assert get_resp_header(conn, "etag") == [~s{"digest"}]
end
test "serves the compressed file if available" do
conn =
conn(:get, "/app.js")
|> put_req_header("accept-encoding", "gzip")
|> call()
assert conn.status == 200
assert get_resp_header(conn, "content-encoding") == ["gzip"]
assert get_resp_header(conn, "content-type") == ["text/javascript"]
assert get_resp_header(conn, "etag") == [~s{"digest"}]
end
test "ignores unavailable paths" do
conn =
conn(:get, "/invalid.js")
|> call()
assert conn.status == 404
end
end