Migrate to esbuild (#1934)

.github/workflows/deploy.yml

@@ -44,7 +44,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Clean generated assets
-        run: rm -rf static/{js,css}
+        run: rm -rf static/assets
       - name: Install npm dependencies
         run: npm ci --prefix assets
       - name: Build assets

README.md

@@ -270,7 +270,7 @@ Livebook is primarily a Phoenix web application and can be setup as such:
 ```shell
 git clone https://github.com/livebook-dev/livebook.git
 cd livebook
-mix dev.setup
+mix setup
 
 # Run the Livebook server
 mix phx.server

assets/build.js

@@ -0,0 +1,69 @@
+const path = require("path");
+const esbuild = require("esbuild");
+const stylePlugin = require("esbuild-style-plugin");
+const postcssImport = require("postcss-import");
+const tailwind = require("tailwindcss");
+const autoprefixer = require("autoprefixer");
+
+const args = process.argv.slice(2);
+const watch = args.includes("--watch");
+const deploy = args.includes("--deploy");
+
+const outDir = path.resolve(
+  __dirname,
+  deploy ? "../static/assets" : "../tmp/static_dev/assets"
+);
+
+async function main() {
+  await esbuild.build({
+    entryPoints: [
+      "./node_modules/monaco-editor/esm/vs/language/json/json.worker.js",
+      "./node_modules/monaco-editor/esm/vs/editor/editor.worker.js",
+    ],
+    outdir: outDir,
+    bundle: true,
+    target: "es2017",
+    format: "iife",
+    minify: deploy,
+  });
+
+  const ctx = await esbuild.context({
+    entryPoints: ["js/app.js"],
+    outdir: outDir,
+    bundle: true,
+    splitting: true,
+    target: "es2017",
+    format: "esm",
+    minify: deploy,
+    sourcemap: deploy ? undefined : "linked",
+    plugins: [
+      stylePlugin({
+        postcss: {
+          plugins: [postcssImport, tailwind, autoprefixer],
+        },
+      }),
+    ],
+    loader: {
+      ".ttf": "file",
+      ".woff": "file",
+      ".woff2": "file",
+      ".eot": "file",
+      ".svg": "file",
+    },
+  });
+
+  if (watch) {
+    await ctx.watch();
+
+    process.stdin.on("close", () => {
+      process.exit(0);
+    });
+
+    process.stdin.resume();
+  } else {
+    await ctx.rebuild();
+    process.exit(0);
+  }
+}
+
+main();

assets/js/hooks/audio_input.js

@@ -231,7 +231,7 @@ const AudioInput = {
     }
 
     // If the server uses different endianness, we swap bytes accordingly
-    for (const i = 0; i < buffer.byteLength / 4; i++) {
+    for (let i = 0; i < buffer.byteLength / 4; i++) {
       const b1 = buffer[i];
       const b2 = buffer[i + 1];
       const b3 = buffer[i + 2];

assets/js/hooks/cell_editor/live_editor/monaco.js

@@ -1,4 +1,102 @@
+// For the full list of features see [1] and [2]. The Monaco Webpack
+// plugin always ignores certain imports [3], so we ignore these as
+// well. On top of that, we ignore some other features which we
+// clearly don't use.
+//
+// [1]: https://github.com/microsoft/vscode/blob/main/src/vs/editor/editor.main.ts
+// [2]: https://github.com/microsoft/vscode/blob/main/src/vs/editor/editor.all.ts
+// [3]: https://github.com/microsoft/monaco-editor/blob/v0.38.0/build/releaseMetadata.ts#L212-L221
+
+// === Monaco features ===
+
+import "monaco-editor/esm/vs/editor/browser/coreCommands";
+// import 'monaco-editor/esm/vs/editor/browser/widget/codeEditorWidget';
+// import 'monaco-editor/esm/vs/editor/browser/widget/diffEditorWidget';
+// import 'monaco-editor/esm/vs/editor/browser/widget/diffNavigator';
+import "monaco-editor/esm/vs/editor/contrib/anchorSelect/browser/anchorSelect";
+import "monaco-editor/esm/vs/editor/contrib/bracketMatching/browser/bracketMatching";
+import "monaco-editor/esm/vs/editor/contrib/caretOperations/browser/caretOperations";
+import "monaco-editor/esm/vs/editor/contrib/caretOperations/browser/transpose";
+import "monaco-editor/esm/vs/editor/contrib/clipboard/browser/clipboard";
+import "monaco-editor/esm/vs/editor/contrib/codeAction/browser/codeActionContributions";
+// import 'monaco-editor/esm/vs/editor/contrib/codelens/browser/codelensController';
+// import 'monaco-editor/esm/vs/editor/contrib/colorPicker/browser/colorContributions';
+// import 'monaco-editor/esm/vs/editor/contrib/colorPicker/browser/standaloneColorPickerActions';
+import "monaco-editor/esm/vs/editor/contrib/comment/browser/comment";
+import "monaco-editor/esm/vs/editor/contrib/contextmenu/browser/contextmenu";
+import "monaco-editor/esm/vs/editor/contrib/cursorUndo/browser/cursorUndo";
+import "monaco-editor/esm/vs/editor/contrib/dnd/browser/dnd";
+// import 'monaco-editor/esm/vs/editor/contrib/dropOrPasteInto/browser/copyPasteContribution';
+// import 'monaco-editor/esm/vs/editor/contrib/dropOrPasteInto/browser/dropIntoEditorContribution';
+import "monaco-editor/esm/vs/editor/contrib/find/browser/findController";
+import "monaco-editor/esm/vs/editor/contrib/folding/browser/folding";
+import "monaco-editor/esm/vs/editor/contrib/fontZoom/browser/fontZoom";
+import "monaco-editor/esm/vs/editor/contrib/format/browser/formatActions";
+// import 'monaco-editor/esm/vs/editor/contrib/documentSymbols/browser/documentSymbols';
+import "monaco-editor/esm/vs/editor/contrib/inlineCompletions/browser/inlineCompletions.contribution";
+import "monaco-editor/esm/vs/editor/contrib/inlineProgress/browser/inlineProgress";
+import "monaco-editor/esm/vs/editor/contrib/gotoSymbol/browser/goToCommands";
+import "monaco-editor/esm/vs/editor/contrib/gotoSymbol/browser/link/goToDefinitionAtPosition";
+import "monaco-editor/esm/vs/editor/contrib/gotoError/browser/gotoError";
+import "monaco-editor/esm/vs/editor/contrib/hover/browser/hover";
+import "monaco-editor/esm/vs/editor/contrib/indentation/browser/indentation";
+import "monaco-editor/esm/vs/editor/contrib/inlayHints/browser/inlayHintsContribution";
+import "monaco-editor/esm/vs/editor/contrib/inPlaceReplace/browser/inPlaceReplace";
+import "monaco-editor/esm/vs/editor/contrib/lineSelection/browser/lineSelection";
+import "monaco-editor/esm/vs/editor/contrib/linesOperations/browser/linesOperations";
+import "monaco-editor/esm/vs/editor/contrib/linkedEditing/browser/linkedEditing";
+import "monaco-editor/esm/vs/editor/contrib/links/browser/links";
+import "monaco-editor/esm/vs/editor/contrib/longLinesHelper/browser/longLinesHelper";
+import "monaco-editor/esm/vs/editor/contrib/multicursor/browser/multicursor";
+import "monaco-editor/esm/vs/editor/contrib/parameterHints/browser/parameterHints";
+import "monaco-editor/esm/vs/editor/contrib/rename/browser/rename";
+import "monaco-editor/esm/vs/editor/contrib/semanticTokens/browser/documentSemanticTokens";
+import "monaco-editor/esm/vs/editor/contrib/semanticTokens/browser/viewportSemanticTokens";
+import "monaco-editor/esm/vs/editor/contrib/smartSelect/browser/smartSelect";
+import "monaco-editor/esm/vs/editor/contrib/snippet/browser/snippetController2";
+import "monaco-editor/esm/vs/editor/contrib/stickyScroll/browser/stickyScrollContribution";
+import "monaco-editor/esm/vs/editor/contrib/suggest/browser/suggestController";
+import "monaco-editor/esm/vs/editor/contrib/suggest/browser/suggestInlineCompletions";
+// import 'monaco-editor/esm/vs/editor/contrib/tokenization/browser/tokenization';
+// import 'monaco-editor/esm/vs/editor/contrib/toggleTabFocusMode/browser/toggleTabFocusMode';
+import "monaco-editor/esm/vs/editor/contrib/unicodeHighlighter/browser/unicodeHighlighter";
+import "monaco-editor/esm/vs/editor/contrib/unusualLineTerminators/browser/unusualLineTerminators";
+import "monaco-editor/esm/vs/editor/contrib/wordHighlighter/browser/wordHighlighter";
+import "monaco-editor/esm/vs/editor/contrib/wordOperations/browser/wordOperations";
+import "monaco-editor/esm/vs/editor/contrib/wordPartOperations/browser/wordPartOperations";
+import "monaco-editor/esm/vs/editor/contrib/readOnlyMessage/browser/contribution";
+
+// import 'monaco-editor/esm/vs/editor/common/standaloneStrings';
+// import 'vs/base/browser/ui/codicons/codiconStyles';
+
+import "monaco-editor/esm/vs/editor/standalone/browser/accessibilityHelp/accessibilityHelp";
+import "monaco-editor/esm/vs/editor/standalone/browser/iPadShowKeyboard/iPadShowKeyboard";
+import "monaco-editor/esm/vs/editor/standalone/browser/inspectTokens/inspectTokens";
+import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneHelpQuickAccess";
+import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneGotoLineQuickAccess";
+import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneGotoSymbolQuickAccess";
+import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneCommandsQuickAccess";
+import "monaco-editor/esm/vs/editor/standalone/browser/referenceSearch/standaloneReferenceSearch";
+// import 'monaco-editor/esm/vs/editor/standalone/browser/toggleHighContrast/toggleHighContrast';
+
 import * as monaco from "monaco-editor/esm/vs/editor/editor.api";
+
+// === Languages ===
+
+// Note: JSON doesn't have a monarch grammar in basic-languages, so
+// we use the advanced handler (which also has its own web worker)
+import "monaco-editor/esm/vs/language/json/monaco.contribution";
+
+import "monaco-editor/esm/vs/basic-languages/elixir/elixir.contribution";
+import "monaco-editor/esm/vs/basic-languages/markdown/markdown.contribution";
+import "monaco-editor/esm/vs/basic-languages/javascript/javascript.contribution";
+import "monaco-editor/esm/vs/basic-languages/sql/sql.contribution";
+import "monaco-editor/esm/vs/basic-languages/css/css.contribution";
+import "monaco-editor/esm/vs/basic-languages/html/html.contribution";
+import "monaco-editor/esm/vs/basic-languages/xml/xml.contribution";
+
+// === Configuration ===
+
 import { CommandsRegistry } from "monaco-editor/esm/vs/platform/commands/common/commands";
 import ElixirOnTypeFormattingEditProvider from "./elixir/on_type_formatting_edit_provider";
 import { theme, lightTheme } from "./theme";
@@ -7,6 +105,19 @@ import { PieceTreeTextBufferBuilder } from "monaco-editor/esm/vs/editor/common/m
 
 import { settingsStore } from "../../../lib/settings";
 
+window.MonacoEnvironment = {
+  // Certain browser extensions are Monaco-aware, so we expose it on
+  // the window object
+  globalAPI: true,
+  getWorkerUrl(_workerId, label) {
+    if (label === "json") {
+      return "/assets/language/json/json.worker.js";
+    }
+
+    return "/assets/editor/editor.worker.js";
+  },
+};
+
 // Force LF for line ending.
 //
 // Monaco infers EOL based on the text content if any, otherwise uses

assets/js/lib/markdown/mermaid.js

@@ -37,10 +37,7 @@ export function renderMermaid(definition) {
 }
 
 function importMermaid() {
-  return import(
+  return import("mermaid").then(({ default: mermaid }) => {
-    /* webpackChunkName: "mermaid" */
-    "mermaid"
-  ).then(({ default: mermaid }) => {
     if (!mermaidInitialized) {
       mermaid.initialize({ startOnLoad: false });
       mermaidInitialized = true;

assets/package.json

@@ -1,10 +1,10 @@
 {
   "private": true,
   "scripts": {
-    "deploy": "NODE_ENV=production webpack --mode production",
+    "deploy": "node build.js --deploy",
-    "watch": "webpack --mode development --watch",
+    "watch": "node build.js --watch",
-    "format": "prettier --trailing-comma es5 --write '{js,test,css}/**/*.{js,json,css,scss,md}' --no-error-on-unmatched-pattern",
+    "format": "prettier --trailing-comma es5 --write '{js,test,css}/**/*.{js,json,css,md}' --no-error-on-unmatched-pattern",
-    "format-check": "prettier --trailing-comma es5 --check '{js,test,css}/**/*.{js,json,css,scss,md}' --no-error-on-unmatched-pattern",
+    "format-check": "prettier --trailing-comma es5 --check '{js,test,css}/**/*.{js,json,css,md}' --no-error-on-unmatched-pattern",
     "test": "jest",
     "test:watch": "jest --watch"
   },
@@ -13,11 +13,8 @@
     "@fontsource/jetbrains-mono": "^5.0.1",
     "@fontsource/red-hat-text": "^5.0.1",
     "@picmo/popup-picker": "^5.7.6",
-    "babel-loader": "^9.1.0",
     "crypto-js": "^4.0.0",
-    "css-minimizer-webpack-plugin": "^5.0.0",
     "hast-util-to-text": "^3.1.1",
-    "html-loader": "^4.1.0",
     "hyperlist": "^1.0.0",
     "jest": "^29.1.2",
     "mermaid": "^10.0.2",
@@ -27,8 +24,6 @@
     "phoenix_html": "file:../deps/phoenix_html",
     "phoenix_live_view": "file:../deps/phoenix_live_view",
     "picmo": "^5.7.2",
-    "postcss-import": "^15.0.0",
-    "postcss-loader": "^7.0.1",
     "rehype-katex": "^6.0.0",
     "rehype-parse": "^8.0.3",
     "rehype-raw": "^6.0.0",
@@ -44,18 +39,16 @@
     "topbar": "^2.0.1",
     "unified": "^10.1.0",
     "unist-util-remove-position": "^4.0.1",
-    "unist-util-visit": "^4.0.0",
+    "unist-util-visit": "^4.0.0"
-    "webpack-cli": "^5.0.1"
   },
   "devDependencies": {
-    "@babel/core": "^7.14.0",
+    "@babel/core": "^7.22.1",
-    "@babel/preset-env": "^7.14.1",
+    "@babel/preset-env": "^7.22.2",
     "autoprefixer": "^10.2.0",
-    "css-loader": "^6.4.0",
+    "esbuild": "^0.17.19",
-    "mini-css-extract-plugin": "^2.1.0",
+    "esbuild-style-plugin": "^1.6.2",
-    "monaco-editor-webpack-plugin": "^7.0.1",
     "postcss": "^8.2.3",
-    "prettier": "^2.2.1",
+    "postcss-import": "^15.1.0",
-    "webpack": "^5.37.0"
+    "prettier": "^2.2.1"
   }
 }

assets/postcss.config.js

@@ -1,7 +0,0 @@
-module.exports = {
-  plugins: {
-    "postcss-import": {},
-    tailwindcss: {},
-    autoprefixer: {},
-  },
-};

assets/webpack.config.js

@@ -1,70 +0,0 @@
-const path = require("path");
-const glob = require("glob");
-const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
-const MonacoWebpackPlugin = require("monaco-editor-webpack-plugin");
-
-// Make sure NODE_ENV is set, so that @tailwindcss/jit is in watch mode in development.
-process.env.NODE_ENV = process.env.NODE_ENV || "development";
-
-module.exports = (env, options) => {
-  const devMode = options.mode !== "production";
-
-  return {
-    mode: options.mode || "production",
-    entry: {
-      app: glob.sync("./vendor/**/*.js").concat(["./js/app.js"]),
-    },
-    output: {
-      filename: "[name].js",
-      path: path.resolve(
-        __dirname,
-        devMode ? "../tmp/static_dev/js" : "../static/js"
-      ),
-      publicPath: 'auto',
-    },
-    devtool: devMode ? "eval-cheap-module-source-map" : undefined,
-    module: {
-      rules: [
-        {
-          test: /\.js$/,
-          exclude: /node_modules/,
-          use: {
-            loader: "babel-loader",
-          },
-        },
-        {
-          test: /\.[s]?css$/,
-          use: [MiniCssExtractPlugin.loader, "css-loader", "postcss-loader"],
-        },
-        {
-          test: /\.(ttf|woff|woff2|eot|svg)$/,
-          type: "asset/resource",
-        },
-        {
-          test: /\.html$/i,
-          loader: "html-loader",
-        },
-      ],
-    },
-    plugins: [
-      new MiniCssExtractPlugin({ filename: "../css/app.css" }),
-      new MonacoWebpackPlugin({
-        languages: [
-          "markdown",
-          "elixir",
-          "xml",
-          "json",
-          "sql",
-          "html",
-          "css",
-          "javascript",
-        ],
-        globalAPI: true,
-      }),
-    ],
-    optimization: {
-      minimizer: ["...", new CssMinimizerPlugin()],
-    },
-  };
-};

config/dev.exs

@@ -13,14 +13,7 @@ config :livebook, LivebookWeb.Endpoint,
   debug_errors: true,
   check_origin: false,
   watchers: [
-    node: [
+    npm: ["run", "watch", cd: Path.expand("../assets", __DIR__)]
-      "node_modules/webpack/bin/webpack.js",
-      "--mode",
-      "development",
-      "--watch",
-      "--watch-options-stdin",
-      cd: Path.expand("../assets", __DIR__)
-    ]
   ]
 
 config :livebook, :iframe_port, 4001

lib/livebook_web.ex

@@ -1,7 +1,7 @@
 defmodule LivebookWeb do
   @moduledoc false
 
-  def static_paths, do: ~w(js css fonts images favicon.svg favicon.png robots.txt)
+  def static_paths, do: ~w(assets images favicon.svg favicon.png robots.txt)
 
   def controller do
     quote do

lib/livebook_web/components/layouts/root.html.heex

@@ -10,11 +10,13 @@
     <.live_title>
       <%= assigns[:page_title] || "Livebook" %>
     </.live_title>
-    <link phx-track-static rel="stylesheet" href={~p"/css/app.css"} />
+    <link phx-track-static rel="stylesheet" href={~p"/assets/app.css"} />
     <script>
       window.LIVEBOOK_BASE_URL_PATH = "<%= Livebook.Config.base_url_path() %>";
     </script>
-    <script phx-track-static defer type="text/javascript" src={~p"/js/app.js"}>
+    <%!-- This prevents the script to be loaded twice in Chrome --%>
+    <link rel="modulepreload" href={~p"/assets/app.js"} />
+    <script phx-track-static type="module" type="text/javascript" src={~p"/assets/app.js"}>
     </script>
   </head>
   <body

lib/livebook_web/controllers/error_html/404.html.heex

@@ -7,7 +7,7 @@
     <link rel="icon" type="image/svg+xml" href={~p"/favicon.svg"} />
     <link rel="alternate icon" type="image/png" href={~p"/favicon.png"} />
     <title><%= @status %> - Livebook</title>
-    <link rel="stylesheet" href={~p"/css/app.css"} />
+    <link rel="stylesheet" href={~p"/assets/app.css"} />
   </head>
   <body>
     <div class="h-screen flex items-center justify-center bg-gray-900">

lib/livebook_web/controllers/error_html/500.html.heex

@@ -7,7 +7,7 @@
     <link rel="icon" type="image/svg+xml" href={~p"/favicon.svg"} />
     <link rel="alternate icon" type="image/png" href={~p"/favicon.png"} />
     <title><%= @status %> - Livebook</title>
-    <link rel="stylesheet" href={~p"/css/app.css"} />
+    <link rel="stylesheet" href={~p"/assets/app.css"} />
   </head>
   <body>
     <div class="h-screen flex items-center justify-center bg-gray-900">

mix.exs

@@ -51,9 +51,9 @@ defmodule Livebook.MixProject do
 
   defp aliases do
     [
-      "dev.setup": ["deps.get", "cmd npm install --prefix assets"],
+      setup: ["deps.get", "cmd npm install --prefix assets"],
-      "dev.build": ["cmd npm run deploy --prefix ./assets"],
+      "assets.deploy": ["cmd npm run deploy --prefix assets"],
-      "format.all": ["format", "cmd npm run format --prefix ./assets"],
+      "format.all": ["format", "cmd npm run format --prefix assets"],
       "protobuf.generate": ["cmd --cd proto mix protobuf.generate"]
     ]
   end