livebook-dev/livebook
1
1
Fork 0
mirror of https://github.com/livebook-dev/livebook.git synced 2025-11-08 05:04:46 +08:00
Code Issues Projects Releases Packages Wiki Activity

Improve code docs

Browse source
This commit is contained in:
Jonatan Kłosko 2023-07-28 02:04:50 +02:00
parent 4b6d01b4aa
commit 32a98ff0af
35 changed files with 370 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/livebook.ex
View file

@ -70,6 +70,7 @@ defmodule Livebook do
path: "/path/to/other_notebook.livemd"
}
]
"""
@doc """

2
lib/livebook/application.ex
View file

@ -34,7 +34,7 @@ defmodule Livebook.Application do
{DynamicSupervisor, name: Livebook.SessionSupervisor, strategy: :one_for_one},
# Start the server responsible for associating files with sessions
Livebook.Session.FileGuard,
# Start the Node Pool for managing node names
# Start the node pool for managing node names
Livebook.Runtime.NodePool,
# Start the unique task dependencies
Livebook.Utils.UniqueTask,

42
lib/livebook/delta.ex
View file

@ -1,23 +1,23 @@
defmodule Livebook.Delta do
@moduledoc false
# Delta is a format used to represent a set of changes
# introduced to a text document.
# Delta is a format used to represent a set of changes introduced
# to a text document.
#
# By design, delta is suitable for Operational Transformation
# and is hence our primary building block in collaborative text editing.
# By design, delta is suitable for Operational Transformation and
# is hence our primary building block in collaborative text editing.
#
# For a detailed write-up see https://quilljs.com/docs/delta
# and https://quilljs.com/guides/designing-the-delta-format.
# The specification covers rich-text editing, while we only
# need to work with plain-text, so we use a subset of the specification
# For a detailed write-up see https://quilljs.com/docs/delta and
# https://quilljs.com/guides/designing-the-delta-format. The
# specification covers rich-text editing, while we only need to
# work with plain-text, so we use a subset of the specification
# with operations listed in `Livebook.Delta.Operation`.
#
# An implementation of the full Delta specification is available
# in the :text_delta package (https://github.com/deltadoc/text_delta)
# An implementation of the full Delta specification is available in
# the :text_delta package (https://github.com/deltadoc/text_delta)
# by Konstantin Kudryashov under the MIT license. This module builds
# directly on that package, and is simplified to better fit our
# not rich-text use case.
# directly on that package, and is simplified to better fit our not
# rich-text use case.
defstruct ops: []
@ -68,13 +68,19 @@ defmodule Livebook.Delta do
The specification imposes two constraints:
1. Delta must be *compact* - there must be no shorter equivalent delta.
2. Delta must be *canonical* - there is just a single valid representation of the given change.
1. Delta must be *compact* - there must be no shorter equivalent
delta.
2. Delta must be *canonical* - there is just a single valid
representation of the given change.
To satisfy these constraints we follow two rules:
1. Delete followed by insert is swapped to ensure that insert goes first.
2. Operations of the same type are merged.
1. Delete followed by insert is swapped to ensure that insert
goes first.
2. Operations of the same type are merged.
"""
@spec append(t(), Operation.t()) :: t()
def append(delta, op) do
@ -130,8 +136,8 @@ defmodule Livebook.Delta do
end
@doc """
Converts the given delta to a compact representation,
suitable for sending over the network.
Converts the given delta to a compact representation, suitable for
sending over the network.
## Examples

16
lib/livebook/delta/operation.ex
View file

@ -5,9 +5,15 @@ defmodule Livebook.Delta.Operation do
#
# For plain-text (our use case) an operation can be either of:
#
# * `{:insert, string}` - insert the given text at the current position
# * `{:retain, length}` - preserve the given number of characters (effectively moving the cursor)
# * `{:delete, number}` - delete the given number of characters starting from the current position
# * `{:insert, string}` - insert the given text at the current
# position
#
# * `{:retain, length}` - preserve the given number of characters
# (effectively moving the cursor)
#
# * `{:delete, number}` - delete the given number of characters
# starting from the current position
#
import Kernel, except: [length: 1]
@ -72,8 +78,8 @@ defmodule Livebook.Delta.Operation do
def from_compressed(length) when is_integer(length) and length < 0, do: {:delete, -length}
@doc """
Modifies the given operation lists, so that their heads
have the same operation length.
Modifies the given operation lists, so that their heads have the
same operation length.
## Examples

41
lib/livebook/delta/transformation.ex
View file

@ -1,22 +1,25 @@
defmodule Livebook.Delta.Transformation do
@moduledoc false
# Implementation of the Operational Transformation concept for deltas.
# Implementation of the Operational Transformation algorithm for
# deltas.
#
# The transformation allows for conflict resolution in concurrent editing.
# Consider delta `Oa` and delta `Ob` that occurred at the same time against the same text state `S`.
# The resulting new text states are `S ∘ Oa` and `S ∘ Ob` respectively.
# Now for each text state we would like to apply the other delta,
# so that both texts converge to the same state, i.e.:
# The transformation allows for conflict resolution in concurrent
# editing. Consider delta `Oa` and delta `Ob` that occurred at the
# same time against the same text state `S`. The resulting new text
# states are `S ∘ Oa` and `S ∘ Ob` respectively. Now for each text
# state we would like to apply the other delta, so that both texts
# converge to the same state, that is:
#
# `S ∘ Oa ∘ transform(Oa, Ob) = S ∘ Ob ∘ transform(Ob, Oa)`
# S ∘ Oa ∘ transform(Oa, Ob) = S ∘ Ob ∘ transform(Ob, Oa)
#
# That's the high-level idea.
# To actually achieve convergence we have to introduce a linear order of operations.
# This way we can resolve conflicts - e.g. if two deltas insert a text at the same
# position, we have to unambiguously determine which takes precedence.
# A reasonable solution is to have a server process where all
# the clients send deltas, as it naturally imposes the necessary ordering.
# That's the high-level idea. To actually achieve convergence we
# have to introduce a linear order of operations. This way we can
# resolve conflicts, for example, if two deltas insert a text at
# the same position, we have to unambiguously determine which takes
# precedence. A reasonable solution is to have a server process where
# all the clients send deltas, as it naturally imposes the necessary
# ordering.
alias Livebook.Delta
alias Livebook.Delta.Operation
@ -26,13 +29,13 @@ defmodule Livebook.Delta.Transformation do
@doc """
Transforms `right` delta against the `left` delta.
Assuming both deltas represent changes applied to the same
document state, this operation results in modified `right` delta
that represents effectively the same changes (preserved intent),
but works on the document with `left` delta already applied.
Assuming both deltas represent changes applied to the same document
state, this operation results in modified `right` delta that represents
effectively the same changes (preserved intent), but works on the
document with `left` delta already applied.
The `priority` indicates which delta is considered to have
happened first and is used for conflict resolution.
The `priority` indicates which delta is considered to have happened
first and is used for conflict resolution.
"""
@spec transform(Delta.t(), Delta.t(), priority()) :: Delta.t()
def transform(left, right, priority) do

1
lib/livebook/ecto_types/hex_color.ex
View file

@ -1,5 +1,6 @@
defmodule Livebook.EctoTypes.HexColor do
@moduledoc false
use Ecto.Type
@impl true

75
lib/livebook/file_system.ex
View file

@ -1,8 +1,8 @@
defprotocol Livebook.FileSystem do
@moduledoc false
# This protocol defines an interface for file systems
# that can be plugged into Livebook.
# This protocol defines an interface for a virtual file system that
# can be plugged into Livebook.
@typedoc """
An identifier uniquely identifying the given file system.
@ -12,24 +12,24 @@ defprotocol Livebook.FileSystem do
@type id :: String.t()
@typedoc """
A path uniquely idenfies file in the file system.
A path uniquely identifies file in the file system.
Path has most of the semantics of regular file paths,
with the following exceptions:
Path has most of the semantics of regular file paths, with the
following exceptions:
* path must be be absolute for consistency
* directory path must have a trailing slash, whereas
regular file path must not have a trailing slash.
Rationale: some file systems allow a directory and
a file with the same name to co-exist, while path
needs to distinguish between them
* directory path must have a trailing slash, whereas regular file
path must not have a trailing slash. Rationale: certain file
systems allow a directory and a file with the same name to
co-exist, while path needs to distinguish between them
"""
@type path :: String.t()
@typedoc """
A human-readable error message clarifying the operation
failure reason.
A human-readable error message clarifying the operation failure
reason.
"""
@type error :: String.t()
@ -49,8 +49,8 @@ defprotocol Livebook.FileSystem do
* `:local` - if the resource is local to its node
* `:global` - if the resource is external and accessible
from any node
* `:global` - if the resource is external and accessible from any
node
"""
@spec type(t()) :: :local | :global
@ -59,9 +59,9 @@ defprotocol Livebook.FileSystem do
@doc """
Returns the default directory path.
To some extent this is similar to current working directory
in a regular file system. For most file systems this
will just be the root path.
To some extent this is similar to current working directory in a
regular file system. For most file systems this will just be the
root path.
"""
@spec default_path(t()) :: path()
def default_path(file_system)
@ -69,8 +69,8 @@ defprotocol Livebook.FileSystem do
@doc """
Returns a list of files located in the given directory.
When `recursive` is set to `true`, nested directories
are traversed and the final list includes all the paths.
When `recursive` is set to `true`, nested directories are traversed
and the final list includes all the paths.
"""
@spec list(t(), path(), boolean()) :: {:ok, list(path())} | {:error, error()}
def list(file_system, path, recursive)
@ -86,8 +86,8 @@ defprotocol Livebook.FileSystem do
If the file exists, it gets overridden.
If the file doesn't exist, it gets created along with
all the necessary directories.
If the file doesn't exist, it gets created along with all the
necessary directories.
"""
@spec write(t(), path(), binary()) :: :ok | {:error, error()}
def write(file_system, path, content)
@ -95,9 +95,9 @@ defprotocol Livebook.FileSystem do
@doc """
Returns the current access level to the given file.
If determining the access is costly, then this function may
always return the most liberal access, since all access
functions return error on an invalid attempt.
If determining the access is costly, then this function may always
return the most liberal access, since all access functions return
error on an invalid attempt.
"""
@spec access(t(), path()) :: {:ok, access()} | {:error, error()}
def access(file_system, path)
@ -113,8 +113,7 @@ defprotocol Livebook.FileSystem do
@doc """
Removes the given file.
If a directory is given, all of its contents are removed
recursively.
If a directory is given, all of its contents are removed recursively.
If the file doesn't exist, no error is returned.
"""
@ -126,8 +125,8 @@ defprotocol Livebook.FileSystem do
The given files must be of the same type.
If regular files are given, the contents are copied,
potentially overriding the destination if it already exists.
If regular files are given, the contents are copied, potentially
overriding the destination if it already exists.
If directories are given, the directory contents are copied
recursively.
@ -138,8 +137,8 @@ defprotocol Livebook.FileSystem do
@doc """
Renames the given file.
If a directory is given, it gets renamed as expected and
consequently all of the child paths change.
If a directory is given, it gets renamed as expected and consequently
all of the child paths change.
If the destination exists, an error is returned.
"""
@ -149,9 +148,9 @@ defprotocol Livebook.FileSystem do
@doc """
Returns a version identifier for the given file.
The resulting value must be a string of ASCII characters
placed between double quotes, suitable for use as the
value of the ETag HTTP header.
The resulting value must be a string of ASCII characters placed
between double quotes, suitable for use as the value of the ETag
HTTP header.
"""
@spec etag_for(t(), path()) :: {:ok, String.t()} | {:error, error()}
def etag_for(file_system, path)
@ -165,12 +164,12 @@ defprotocol Livebook.FileSystem do
@doc """
Resolves `subject` against a valid directory path.
The `subject` may be either relative or absolute,
contain special sequences such as ".." and ".",
but the interpretation is left up to the file system.
The `subject` may be either relative or absolute, contain special
sequences such as ".." and ".", but the interpretation is left up
to the file system.
In other words, this has the semantics of path join
followed by expand.
In other words, this has the semantics of path join followed by
expand.
"""
@spec resolve_path(t(), path(), String.t()) :: path()
def resolve_path(file_system, dir_path, subject)

47
lib/livebook/file_system/file.ex
View file

@ -1,13 +1,11 @@
defmodule Livebook.FileSystem.File do
@moduledoc false
# A file points to a specific location in the given
# file system.
# A file points to a specific location in the given file system.
#
# This module provides a number of high-level functions
# similar to the `File` and `Path` core module. Many
# functions simply delegate the work to the underlying
# file system.
# This module provides a number of high-level functions similar to
# the `File` and `Path` core module. Many functions simply delegate
# the work to the underlying file system.
defstruct [:file_system, :path]
@ -44,8 +42,8 @@ defmodule Livebook.FileSystem.File do
end
@doc """
Returns a new file within the `Livebook.FileSystem.Local`
file system.
Returns a new file within the `Livebook.FileSystem.Local` file
system.
"""
@spec local(FileSystem.path()) :: t()
def local(path) do
@ -53,8 +51,8 @@ defmodule Livebook.FileSystem.File do
end
@doc """
Returns a term uniquely identifying the file together
with its file system.
Returns a term uniquely identifying the file together with its file
system.
"""
@spec resource_identifier(t()) :: term()
def resource_identifier(file) do
@ -70,11 +68,10 @@ defmodule Livebook.FileSystem.File do
end
@doc """
Returns a new file resulting from resolving `subject`
against `file`.
Returns a new file resulting from resolving `subject` against `file`.
An absolute path may be given, in which case it
replaces the file path altogether.
An absolute path may be given, in which case it replaces the file
path altogether.
"""
@spec resolve(t(), String.t()) :: t()
def resolve(file, subject) do
@ -114,8 +111,8 @@ defmodule Livebook.FileSystem.File do
@doc """
Returns a directory that contains the given file.
If a directory is given, the parent directory is returned.
Root directory is mapped to itself for consistency.
If a directory is given, the parent directory is returned. Root
directory is mapped to itself for consistency.
"""
@spec containing_dir(t()) :: t()
def containing_dir(file) do
@ -137,8 +134,8 @@ defmodule Livebook.FileSystem.File do
## Options
* `:recursive` - whether to traverse all nested directories,
defaults to `false`
* `:recursive` - whether to traverse all nested directories.
Defaults to `false`
"""
@spec list(t(), keyword()) :: {:ok, list(t())} | {:error, FileSystem.error()}
@ -194,10 +191,9 @@ defmodule Livebook.FileSystem.File do
@doc """
Copies the given file or directory contents.
Files from different file systems are supported,
however keep in mind that this involves reading
contents of individual files from one file system
and writing them to the other.
Files from different file systems are supported, however keep in
mind that this copies individual files chunk by chunk from one file
system to the other.
"""
@spec copy(t(), t()) :: :ok | {:error, FileSystem.error()}
def copy(source, destination)
@ -235,10 +231,9 @@ defmodule Livebook.FileSystem.File do
@doc """
Renames the given file.
Files from different file systems are supported,
however keep in mind that this involves reading
contents of individual files from one file system
and writing them to the other.
Files from different file systems are supported, however keep in
mind that this copies individual files chunk by chunk from one file
system to the other.
"""
@spec rename(t(), t()) :: :ok | {:error, FileSystem.error()}
def rename(source, destination)

12
lib/livebook/intellisense.ex
View file

@ -1,11 +1,11 @@
defmodule Livebook.Intellisense do
@moduledoc false
# This module provides intellisense related operations
# suitable for integration with a text editor.
# This module provides intellisense related operations suitable for
# integration with a text editor.
#
# In a way, this provides the very basic features of a
# language server that Livebook uses.
# In a way, this provides the very basic features of a language
# server that Livebook uses.
alias Livebook.Intellisense.{IdentifierMatcher, SignatureMatcher, Docs}
alias Livebook.Runtime
@ -17,8 +17,8 @@ defmodule Livebook.Intellisense do
@typedoc """
Evaluation state to consider for intellisense.
The `:map_binding` is only called when a value needs to
be extracted from binding.
The `:map_binding` is only called when a value needs to be extracted
from binding.
"""
@type context :: %{
env: Macro.Env.t(),

16
lib/livebook/intellisense/docs.ex
View file

@ -61,14 +61,14 @@ defmodule Livebook.Intellisense.Docs do
matching the name are returned.
Functions with default arguments are normalized, such that each
arity is treated as a separate member, sourcing documentation
from the original one.
arity is treated as a separate member, sourcing documentation from
the original one.
## Options
* `:kinds` - a list of member kinds to limit the lookup to.
Valid kinds are `:function`, `:macro` and `:type`. Defaults
to all kinds
* `:kinds` - a list of member kinds to limit the lookup to. Valid
kinds are `:function`, `:macro` and `:type`. Defaults to all
kinds
"""
@spec lookup_module_members(
@ -158,9 +158,9 @@ defmodule Livebook.Intellisense.Docs do
end
end
# In case insensitive file systems, attempting to load
# Elixir will log a warning in the terminal as it wrongly
# loads elixir.beam, so we explicitly list it.
# In case insensitive file systems, attempting to load Elixir will
# log a warning in the terminal as it wrongly loads elixir.beam,
# so we explicitly list it.
defp ensure_loaded?(Elixir), do: false
defp ensure_loaded?(module), do: Code.ensure_loaded?(module)
end

25
lib/livebook/intellisense/identifier_matcher.ex
View file

@ -1,17 +1,16 @@
defmodule Livebook.Intellisense.IdentifierMatcher do
@moduledoc false
# This module allows for extracting information about
# identifiers based on code and runtime information
# (binding, environment).
# This module allows for extracting information about identifiers
# based on code and runtime information (binding, environment).
#
# This functionality is a basic building block to be
# used for code completion and information extraction.
# This functionality is a basic building block to be used for code
# completion and information extraction.
#
# The implementation is based primarily on `IEx.Autocomplete`.
# It also takes insights from `ElixirSense.Providers.Suggestion.Complete`,
# which is a very extensive implementation used in the
# Elixir Language Server.
# The implementation is based primarily on `IEx.Autocomplete`. It
# also takes insights from `ElixirSense.Providers.Suggestion.Complete`,
# which is a very extensive implementation used in the Elixir Language
# Server.
alias Livebook.Intellisense
alias Livebook.Intellisense.Docs
@ -103,8 +102,8 @@ defmodule Livebook.Intellisense.IdentifierMatcher do
@alias_only_charlists ~w(alias import require)c
@doc """
Returns a list of identifiers matching the given `hint`
together with relevant information.
Returns a list of identifiers matching the given `hint` together
with relevant information.
Evaluation binding and environment is used to expand aliases,
imports, nested maps, etc.
@ -126,8 +125,8 @@ defmodule Livebook.Intellisense.IdentifierMatcher do
end
@doc """
Extracts information about an identifier found in `column`
in `line`.
Extracts information about an identifier found in `column` in
`line`.
The function returns range of columns where the identifier
is located and a list of matching identifier items.

4
lib/livebook/intellisense/signature_matcher.ex
View file

@ -1,8 +1,8 @@
defmodule Livebook.Intellisense.SignatureMatcher do
@moduledoc false
# This module allows for extracting information about
# function signatures matching an incomplete call.
# This module allows for extracting information about function
# signatures matching an incomplete call.
alias Livebook.Intellisense.Docs

73
lib/livebook/live_markdown.ex
View file

@ -1,56 +1,63 @@
defmodule Livebook.LiveMarkdown do
@moduledoc false
# Notebook file format used by Livebook.
# File format used to store Livebook notebooks.
#
# The format is based off of Markdown and preserves compatibility,
# in the sense that every LiveMarkdown file is a valid Markdown file.
# LiveMarkdown uses HTML comments for storing metadata, so a Markdown
# standard supporting such comments is assumed. Not every Markdown file
# is a valid LiveMarkdown file, but may be converted to such by applying
# tiny changes, which the import function does.
# The format is a subset of Markdown, which means that every Live
# Markdown is valid Markdown. Live Markdown uses HTML comments for
# storing certain metadata, so we assume a Markdown standard that
# supports comments. On the other hand, not every Markdown is a
# valid Live Markdown file, however it can be imported as such by
# applying tiny changes and assumptions.
#
# Currently the format is straightforward and specifies the following:
# Currently the Live Markdown format imposes the following rules:
#
# 1. The file should have a leading *Heading 1* holding the notebook name
# 1. The file should have a leading *Heading 1* which contains
# the notebook name.
#
# 2. Every *Heading 2* starts a new section
# 2. Every *Heading 2* starts a new section.
#
# 3. Every Elixir code block maps to a Code cell
# 3. Every Elixir/Erlang code block represents a Code cell.
#
# 4. Adjacent regular Markdown text maps to a Markdown cell
# 4. Adjacent regular Markdown blocks represents a Markdown cell.
#
# 5. Comments of the form `<!-- livebook:json_object -->` hold Livebook data
# and may be one of the following:
# 5. Comments of the format `<!-- livebook:json -->` may appear
# anywhere in the file and hold Livebook specific data. The
# data should be treated as opaque, but is either of the
# following:
#
# * description of a notebook object that cannot be naturally encoded
# using Markdown, in such case the JSON contains a "livebook_object" field
# * description of a notebook object that cannot be naturally
# encoded using Markdown, such as Smart cell. In such case
# the JSON contains the `livebook_object` field
#
# * notebook stamp data with `"stamp"` and `"offset"` fields
# * notebook, section or cell metadata
#
# * metadata that may appear anywhere and applies to the element
# it directly precedes, recognised metadatas are:
# * `{"force_markdown":true}` - an annotation forcing the next
# next Markdown block to be treated as part of Markdown cell
# (relevant for Elixir/Erlang code blocks, which otherwise
# are interpreted as Code cells)
#
# - `{"force_markdown":true}` - an annotation forcing the next Markdown
# block to be treated as part of Markdown cell (relevant for Elixir code
# blocks, which otherwise are interpreted as Code cells)
# * `{"break_markdown":true}` - an annotation splitting the
# markdown content into separate Markdown cells
#
# - `{"break_markdown":true}` - an annotation splitting the markdown content
# into separate Markdown cells
# * `{"output":true}` - an annotation marking a code snippet
# as cell output
#
# - `{"output":true}` - an annotation marking a code snippet as cell output
# * notebook stamp data with `"stamp"` and `"offset"` fields,
# placed at the very end of the file. For more details see
# `t:Livebook.Hubs.Provider.notebook_stamp/0`
#
# - section metadata, recognised keys `branch_parent_index`
# 6. An optional code block may appear between the notebook name
# heading and the first section heading. This block is parsed
# as the setup Code cell.
#
# - cell metadata, recognised keys: `disable_formatting`
#
# 6. Any comments before the leading heading are kept.
# 7. Any comments before the leading heading are kept.
#
# ## Example
#
# Here's an example LiveMarkdown file:
#
# # My Notebook
# # My notebook
#
# ## Section 1
#
@ -60,7 +67,7 @@ defmodule Livebook.LiveMarkdown do
# * Elixir
# * PostgreSQL
#
# <!-- livebook:{"readonly":true} -->
# <!-- livebook:{"disable_formatting":true} -->
#
# ```elixir
# Enum.to_list(1..10)
@ -74,8 +81,6 @@ defmodule Livebook.LiveMarkdown do
# # More Elixir code
# ```
#
# This file defines a notebook named *My Notebook* with two sections.
# The first section includes 3 cells and the second section includes 1 Code cell.
@doc """
The file extension used by Live Markdown files.
@ -89,7 +94,7 @@ defmodule Livebook.LiveMarkdown do
* `:include_outputs` - whether to render cell outputs.
Only textual outputs are included. Defaults to the
value of `:persist_outputs` notebook attribute.
value of `:persist_outputs` notebook attribute
"""
@spec notebook_to_livemd(Notebook.t(), keyword()) :: {String.t(), list(String.t())}

43
lib/livebook/notebook.ex
View file

@ -3,19 +3,18 @@ defmodule Livebook.Notebook do
# Data structure representing a notebook.
#
# A notebook is just the representation and roughly
# maps to a file that the user can edit.
# A notebook is just a document and roughly maps to a plain file
# that the user can edit.
#
# A notebook *session* is a living process that holds a specific
# A notebook **session** is a living process that holds a specific
# notebook instance and allows users to collaboratively apply
# changes to this notebook.
# changes to that notebook. See `Livebook.Session`.
#
# A notebook is divided into a number of *sections*, each
# containing a number of *cells*.
# Structurally, a notebook is divided into a number of **sections**,
# each containing a number of **cells**.
defstruct [
:name,
:version,
:setup_section,
:sections,
:leading_comments,
@ -37,7 +36,6 @@ defmodule Livebook.Notebook do
@type t :: %__MODULE__{
name: String.t(),
version: String.t(),
setup_section: Section.t(),
sections: list(Section.t()),
leading_comments: list(list(line :: String.t())),
@ -53,6 +51,32 @@ defmodule Livebook.Notebook do
teams_enabled: boolean()
}
@typedoc """
File entry represents a virtual file that the notebook is aware of.
Files can be of different types:
* `:attachment` - a hard copy of a file managed together with the
notebook. These files are stored in files/ directory alongside
the notebook file
* `:file` - absolute link to a file on any of the available file
systems
* `:url` - absolute link to a file available online
## Quarantine
File entries of type `:file` are somewhat sensitive, since they may
point to an external file system, like S3. When importing a notebook,
we don't want to allow access to arbitrary files, since the user may
not realize what exact location the file entry is pointing to. Hence,
on import we place these file entries in "quarantine" and require
the user to explicitly allow access to them. We persist this choice
using the notebook stamp, so if the file is saved and opened later,
the files are automatically allowed. If the stamp is not valid, all
files are placed back in the quarantine.
"""
@type file_entry ::
%{
name: String.t(),
@ -69,8 +93,6 @@ defmodule Livebook.Notebook do
url: String.t()
}
@version "1.0"
@doc """
Returns a blank notebook.
"""
@ -78,7 +100,6 @@ defmodule Livebook.Notebook do
def new() do
%__MODULE__{
name: "Untitled notebook",
version: @version,
setup_section: %{Section.new() | id: "setup-section", name: "Setup", cells: []},
sections: [],
leading_comments: [],

2
lib/livebook/notebook/app_settings.ex
View file

@ -1,6 +1,8 @@
defmodule Livebook.Notebook.AppSettings do
@moduledoc false
# Data structure configuring how notebook gets deployed as an app.
use Ecto.Schema
import Ecto.Changeset, except: [change: 1, change: 2]

6
lib/livebook/notebook/cell.ex
View file

@ -3,9 +3,9 @@ defmodule Livebook.Notebook.Cell do
# Data structure representing a single cell in a notebook.
#
# A cell is the smallest unit of work in a notebook.
# It may consist of text content, outputs, rendered content
# and other special forms.
# Cell is the smallest structural unit in a notebook, in other words
# it is a block. Depending on the cell type, it may consist of text
# content, outputs or a specific UI.
alias Livebook.Utils
alias Livebook.Notebook.Cell

6
lib/livebook/notebook/cell/code.ex
View file

@ -1,10 +1,10 @@
defmodule Livebook.Notebook.Cell.Code do
@moduledoc false
# A cell with Elixir code.
# Notebook cell with evaluable code.
#
# It consists of text content that the user can edit
# and produces some output once evaluated.
# It consists of text content that the user can edit and produces
# output once evaluated.
defstruct [
:id,

6
lib/livebook/notebook/cell/markdown.ex
View file

@ -1,10 +1,10 @@
defmodule Livebook.Notebook.Cell.Markdown do
@moduledoc false
# A cell with Markdown text content.
# Notebook cell with Markdown text content.
#
# It consists of Markdown content that the user can edit
# and which is then rendered on the page.
# It consists of text content that the user can edit and which is
# rendered on the page.
defstruct [:id, :source]

11
lib/livebook/notebook/cell/smart.ex
View file

@ -1,7 +1,16 @@
defmodule Livebook.Notebook.Cell.Smart do
@moduledoc false
# A cell with Elixir code that is edited through a dedicated UI.
# A cell with evaluable code that is edited through a dedicated UI.
#
# Smart cell is supposed to provide the user with an easy, code-free
# way to achieve a specific task, such as plotting a chart or querying
# a database. The user interacts with smart cell through UI, while
# the smart cell generates plain code to be executed. The user can
# access and take over the underlying code at any point.
#
# The available smart cells come from the runtime, therefore they
# are one Livebook's extension points.
defstruct [:id, :source, :chunks, :outputs, :kind, :attrs, :js_view, :editor]

4
lib/livebook/notebook/content_loader.ex
View file

@ -9,12 +9,14 @@ defmodule Livebook.Notebook.ContentLoader do
@type location :: {:file, FileSystem.File.t()} | {:url, String.t()}
@doc """
Rewrite known URLs, so that they point to plain text file rather than HTML.
Rewrite known URLs, so that they point to plain text file rather
than HTML.
Currently the rewerites handle:
* GitHub files
* Gist files
"""
@spec rewrite_url(String.t()) :: String.t()
def rewrite_url(url) do

10
lib/livebook/notebook/section.ex
View file

@ -3,12 +3,12 @@ defmodule Livebook.Notebook.Section do
# Data structure representing a single section in a notebook.
#
# Each section contains a number of cells and serves as a way
# of grouping related cells.
# Section can contains a number of cells and serves as a way of
# grouping related cells.
#
# A section may optionally have a parent, in which case it's
# a branching section. Such section logically follows its
# parent section and has no impact on any further sections.
# A section may optionally have a parent, in which case it is a
# **branching section**. Such section logically follows its parent
# section and has no impact on any further sections.
defstruct [:id, :name, :cells, :parent_id]

35
lib/livebook/runtime/erl_dist.ex
View file

@ -1,25 +1,28 @@
defmodule Livebook.Runtime.ErlDist do
@moduledoc false
# This module allows for initializing nodes connected using
# Erlang Distribution with modules and processes necessary for evaluation.
# This module allows for initializing connected runtime nodes with
# modules and processes necessary for evaluation.
#
# To ensure proper isolation between sessions,
# code evaluation may take place in a separate Elixir runtime,
# which also makes it easy to terminate the whole
# evaluation environment without stopping Livebook.
# This is what `Runtime.ElixirStandalone` and `Runtime.Attached` do,
# so this module contains the shared functionality they need.
# To ensure proper isolation between sessions, code evaluation may
# take place in a separate Elixir runtime, which also makes it easy
# to terminate the whole evaluation environment without stopping
# Livebook. Both `Runtime.ElixirStandalone` and `Runtime.Attached`
# do that and this module contains the shared functionality.
#
# To work with a separate node, we have to inject the necessary
# Livebook modules there and also start the relevant processes
# related to evaluation. Fortunately Erlang allows us to send modules
# binary representation to the other node and load them dynamically.
# related to evaluation. Fortunately Erlang allows us to send
# modules binary representation to the other node and load them
# dynamically.
#
# For further details see `Livebook.Runtime.ErlDist.NodeManager`.
# Modules to load into the connected node.
def required_modules do
@doc """
Livebook modules necessary for evaluation within a runtime node.
"""
@spec required_modules() :: list(module())
def required_modules() do
[
Livebook.Runtime.Definitions,
Livebook.Runtime.Evaluator,
@ -47,9 +50,8 @@ defmodule Livebook.Runtime.ErlDist do
@doc """
Starts a runtime server on the given node.
If necessary, the required modules are loaded
into the given node and the node manager process
is started with `node_manager_opts`.
If necessary, the required modules are loaded into the given node
and the node manager process is started with `node_manager_opts`.
## Options
@ -85,7 +87,8 @@ defmodule Livebook.Runtime.ErlDist do
if local_otp != remote_otp do
raise RuntimeError,
"failed to load #{inspect(module)} module into the remote node, potentially due to Erlang/OTP version mismatch, reason: #{inspect(reason)} (local #{local_otp} != remote #{remote_otp})"
"failed to load #{inspect(module)} module into the remote node," <>
" potentially due to Erlang/OTP version mismatch, reason: #{inspect(reason)} (local #{local_otp} != remote #{remote_otp})"
else
raise RuntimeError,
"failed to load #{inspect(module)} module into the remote node, reason: #{inspect(reason)}"

29
lib/livebook/runtime/erl_dist/node_manager.ex
View file

@ -3,16 +3,14 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
# The primary Livebook process started on a remote node.
#
# This process is responsible for initializing the node
# with necessary runtime configuration and then starting
# runtime server processes, one per runtime.
# This approach allows for multiple runtimes connected
# to the same node, while preserving the necessary
# cleanup semantics.
# This process is responsible for initializing the node with necessary
# configuration and then starting runtime server processes, one per
# runtime. This approach allows for multiple runtimes connected to
# the same node, while preserving the necessary cleanup semantics.
#
# The manager process terminates as soon as the last runtime
# server terminates. Upon termination the manager reverts the
# runtime configuration back to the initial state.
# The manager process terminates as soon as the last runtime server
# terminates. Upon termination the manager reverts the configuration
# back to the initial state.
use GenServer
@ -26,12 +24,11 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
## Options
* `:unload_modules_on_termination` - whether to unload all
Livebook related modules from the node on termination.
Defaults to `true`
* `:unload_modules_on_termination` - whether to unload all Livebook
related modules from the node on termination. Defaults to `true`
* `:auto_termination` - whether to terminate the manager
when the last runtime server terminates. Defaults to `true`
* `:auto_termination` - whether to terminate the manager when the
last runtime server terminates. Defaults to `true`
* `:parent_node` - indicates which node spawned the node manager.
It is used to disconnect the node when the server terminates,
@ -89,8 +86,8 @@ defmodule Livebook.Runtime.ErlDist.NodeManager do
{:ok, io_proxy_registry} =
Registry.start_link(name: @io_proxy_registry_name, keys: :duplicate)
# Register our own standard error IO device that proxies
# to sender's group leader.
# Register our own standard error IO device that proxies to
# sender's group leader.
original_standard_error = Process.whereis(:standard_error)
{:ok, io_forward_gl_pid} = ErlDist.IOForwardGL.start_link()
Process.unregister(:standard_error)

30
lib/livebook/runtime/evaluator.ex
View file

@ -3,19 +3,18 @@ defmodule Livebook.Runtime.Evaluator do
# A process responsible for evaluating notebook code.
#
# Evaluator receives an evaluation request and synchronously
# evaluates the given code within itself (rather than spawning
# a separate process). It stores the resulting binding and env
# in its state (under a specific reference).
# When evaluator receives an evaluation request, it synchronously
# evaluates the given code within itself, rather than spawning a
# separate process. It stores the resulting binding and env in its
# state (under a specific reference).
#
# Storing the binding in the same process that evaluates the
# code is essential, because otherwise we would have to send it
# to another process, which means copying a potentially massive
# amounts of data.
# Storing the binding in the same process that evaluates the code is
# essential, because otherwise we would have to send it to another
# process, which means copying a potentially massive amounts of data.
#
# Also, note that this process is intentionally not a GenServer,
# Also, note that this process intentionally is not a GenServer,
# because during evaluation we it may receive arbitrary messages
# and we want to keep them in the inbox, while a GenServer would
# and we want to keep them in the inbox, whereas a GenServer would
# always consume them.
require Logger
@ -124,19 +123,18 @@ defmodule Livebook.Runtime.Evaluator do
@doc """
Asynchronously parses and evaluates the given code.
Any exceptions are captured and transformed into an error
result.
The resulting context (binding and env) is stored under `ref`. Any
subsequent calls may specify `parent_refs` pointing to a sequence
of previous evaluations, in which case the corresponding context is
of previous evaluations, in which case the accumulated context is
used as the entry point for evaluation.
Any exceptions are captured and transformed into an error result.
The evaluation result is formatted into an output and sent to the
configured client (see `start_link/1`) together with metadata.
See `Livebook.Runtime.evaluate_code/5` for the messages format
and the list of available options.
See `Livebook.Runtime.evaluate_code/5` for the messages format and
the list of available options.
## Options

17
lib/livebook/runtime/node_pool.ex
View file

@ -3,13 +3,10 @@ defmodule Livebook.Runtime.NodePool do
@moduledoc false
# A pool for reusing child node names.
# A pool with generated node names.
#
# `free_name` refers to the list of unused names.
# `generated_names` refers to the list of names ever generated.
#
# `buffer_time` refers to time the pool waits before
# adding a name to `pool`, which by default is 1 minute.
# The names are randomly generated, however to avoid atom exhaustion
# unused names return back to the pool and can be reused later.
@default_time 60_000
@ -20,11 +17,11 @@ defmodule Livebook.Runtime.NodePool do
## Options
* `:name` - The name the NodePool is locally registered as. By
default, it is `Livebook.Runtime.NodePool`
* `:name` - the name to register the pool process under. Defaults
to `Livebook.Runtime.NodePool`
* `:buffer_time` - The time that is spent before a disconnected
node's name is added to pool. The default is 1 minute.
* `:buffer_time` - the time that is awaited before a disconnected
node's name is added to pool. Defaults to 1 minute
"""
def start_link(opts) do

55
lib/livebook/session.ex
View file

@ -1,25 +1,50 @@
defmodule Livebook.Session do
@moduledoc false
# Server corresponding to a single notebook session.
# Server process representing a single notebook session.
#
# The process keeps the current notebook state and serves
# as a source of truth that multiple clients talk to.
# Receives update requests from the clients and notifies
# them of any changes applied to the notebook.
# Session keeps a notebook document, as well as additional ephemeral
# state, such as evaluation outputs. It serves as a source of truth
# that multiple clients talk to. Those clients send update requests
# or commands to the session, while the session notifies them of any
# changes applied to the notebook state.
#
# ## Collaborative state
#
# The core concept is the `Livebook.Session.Data` structure
# to which we can apply reproducible operations.
# See `Livebook.Session.Data` for more information.
# The core concept is the `%Livebook.Session.Data{}` struct, which
# holds state shared across all of the clients. Refer to the module
# documentation for more details.
#
# ## Runtime
#
# Code evaluation, as well as related features, such as intellisense,
# smart cells and dependency management are abstracted into the
# `Livebook.Runtime` protocol.
#
# Conceptually, we draw a thick line between Livebook server and the
# runtime. In particular, even though Livebook is Elixir centric, it
# rarely makes any Elixir specific assumptions. In theory, the runtime
# could be implemented for another language, as long as it is possible
# to adhere to certain semantics. As a result, the `Livebook.Runtime`
# protocol uses rather generic wording and data types.
#
# ## Evaluation
#
# All regular sections are evaluated in the same process
# (the :main_flow evaluation container). On the other hand,
# each branching section is evaluated in its own process
# and thus runs concurrently.
# The evaluation in Livebook is sequential, that is, all cells in
# regular sections run one by one in the same evaluation container
# (= process) named `:main_flow`. Additionally, Livebook supports
# branching sections. Each branching section forks the evaluation
# state at a certain point and is evaluated in its own container
# (= process) concurrently, while still being a linear continuation
# of the parent section.
#
# The evaluation is sequential, however Livebook is smart about
# reevaluating cells. We keep track of dependencies between cells,
# based on which cells are marked as "stale" and only these cells
# are reevaluated in order to bring the notebook up to date.
#
# For evaluation-specific details refer to `Livebook.Runtime.ErlDist.RuntimeServer`
# and `Livebook.Runtime.Evaluator`.
#
# ### Implementation considerations
#
@ -43,9 +68,9 @@ defmodule Livebook.Session do
# for a single specific evaluation context we make sure to copy
# as little memory as necessary.
# The struct holds the basic session information that we track
# and pass around. The notebook and evaluation state is kept
# within the process state.
# The struct holds the basic session information that we track and
# pass around. The notebook and evaluation state is kept within the
# process state.
defstruct [
:id,
:pid,

16
lib/livebook/session/data.ex
View file

@ -1,18 +1,18 @@
defmodule Livebook.Session.Data do
@moduledoc false
# A structure with shared session data.
# Session data is a state shared across all of the clients.
#
# In practice this structure is a `Notebook` decorated with all
# the ephemeral session data.
# In practice this structure is a `Notebook` decorated with all the
# ephemeral session data.
#
# The data is kept both in the `Session` process and in all client
# processes. All changes go through the `Session` process first to
# introduce linearity and then are broadcasted to the clients, hence
# every client receives changes in the same order. Upon receiving
# an operation, every process applies the change to the locally
# stored `Data`. This way the local `Data` stays the same in all
# processes, while the messages are minimal.
# introduce linearity and then are broadcasted to the clients, so
# that every client receives changes in the same order. Upon
# receiving an operation, every process applies the change to the
# locally stored `%Data{}`. This way the local `%Data{}` stays the
# same in all of the processes, while the messages are minimal.
#
# The operations cover most of the session state management, in
# particular all notebook edits and scheduling cell evaluation.

4
lib/livebook/utils/ansi.ex
View file

@ -31,8 +31,8 @@ defmodule Livebook.Utils.ANSI do
| :light_white
@doc """
Takes a string with ANSI escape codes and parses it
into a list of `{modifiers, string}` parts.
Takes a string with ANSI escape codes and parses it into a list of
`{modifiers, string}` parts.
Also returns the final modifiers.

16
lib/livebook/utils/graph.ex
View file

@ -2,8 +2,8 @@ defmodule Livebook.Utils.Graph do
@moduledoc false
@typedoc """
A bottom-up graph representation encoded as a map
of child-to-parent entries.
A bottom-up graph representation encoded as a map of child-to-parent
entries.
"""
@type t() :: %{node_id => node_id | nil}
@ -14,9 +14,8 @@ defmodule Livebook.Utils.Graph do
@doc """
Finds a path between nodes `from_id` and `to_id`.
If the path exists, a top-down list of nodes is
returned including the extreme nodes. Otherwise,
an empty list is returned.
If the path exists, a top-down list of nodes is returned including
the extreme nodes. Otherwise, an empty list is returned.
"""
@spec find_path(t(), node_id(), node_id()) :: list(node_id())
def find_path(graph, from_id, to_id) do
@ -30,8 +29,7 @@ defmodule Livebook.Utils.Graph do
do: find_path(graph, graph[from_id], to_id, [from_id | path])
@doc """
Finds graph leave nodes, that is, nodes with
no children.
Finds graph leave nodes, that is, nodes with no children.
"""
@spec leaves(t()) :: list(node_id())
def leaves(graph) do
@ -43,8 +41,8 @@ defmodule Livebook.Utils.Graph do
@doc """
Reduces each top-down path in the graph.
Returns a list of accumulators, one for each leaf in the graph,
in no specific order.
Returns a list of accumulators, one for each leaf in the graph, in
no specific order.
"""
@spec reduce_paths(t(), acc, (node_id(), acc -> acc)) :: acc when acc: term()
def reduce_paths(graph, acc, fun) do

4
lib/livebook/utils/http.ex
View file

@ -209,8 +209,8 @@ defmodule Livebook.Utils.HTTP do
pems = :public_key.pem_decode(crt)
ders = Enum.map(pems, fn {:Certificate, der, _} -> der end)
# Note: we need to load the certificates at compilation time,
# as we don't have access to package files in Escript.
# Note: we need to load the certificates at compilation time, as we
# don't have access to package files in Escript.
@cacerts ders
defp http_ssl_opts() do

4
lib/livebook/utils/time.ex
View file

@ -18,8 +18,8 @@ defmodule Livebook.Utils.Time do
end
@doc """
Formats time distance between `from_ndt` and `to_ndt`
as a human-readable string.
Formats time distance between `from_ndt` and `to_ndt` as a
human-readable string.
## Examples

13
lib/livebook/utils/unique_task.ex
View file

@ -45,15 +45,14 @@ defmodule Livebook.Utils.UniqueTask do
end
@doc """
Runs the given function in a separate process,
unless the key is already taken.
Runs the given function in a separate process, unless the key is
already taken.
If another function is already running under the
given key, this call only waits for it to finish
and then returns the same status.
If another function is already running under the given key, this
call only waits for it to finish and then returns the same status.
Returns `:ok` if function finishes successfully and
`:error` if it crashes.
Returns `:ok` if function finishes successfully and `:error` if it
crashes.
"""
@spec run(term(), function()) :: :ok | :error
def run(key, fun) do

4
lib/livebook_web/live/settings_live.ex
View file

@ -123,7 +123,7 @@ defmodule LivebookWeb.SettingsLive do
make system dependencies available to notebooks.
</p>
<.live_component
module={LivebookWeb.EnvVarsComponent}
module={LivebookWeb.SettingsLive.EnvVarsComponent}
id="env-vars"
env_vars={@env_vars}
return_to={~p"/settings"}
@ -196,7 +196,7 @@ defmodule LivebookWeb.SettingsLive do
patch={~p"/settings"}
>
<.live_component
module={LivebookWeb.EnvVarComponent}
module={LivebookWeb.SettingsLive.EnvVarComponent}
id="env-var"
env_var={@env_var}
headline="Configure your application global environment variables."

2
lib/livebook_web/live/env_var_component.ex → lib/livebook_web/live/settings_live/env_var_component.ex
View file

@ -1,4 +1,4 @@
defmodule LivebookWeb.EnvVarComponent do
defmodule LivebookWeb.SettingsLive.EnvVarComponent do
use LivebookWeb, :live_component
alias Livebook.Settings

2
lib/livebook_web/live/env_vars_component.ex → lib/livebook_web/live/settings_live/env_vars_component.ex
View file

@ -1,4 +1,4 @@
defmodule LivebookWeb.EnvVarsComponent do
defmodule LivebookWeb.SettingsLive.EnvVarsComponent do
use LivebookWeb, :live_component
@impl true