mirror of
https://github.com/livebook-dev/livebook.git
synced 2025-10-26 13:27:05 +08:00
Sign S3 requests using aws_signature (#497)
* Sign S3 requests using aws_signature * escape_key -> encode_key * Update lib/livebook/file_system/s3.ex Co-authored-by: José Valim <jose.valim@dashbit.co> Co-authored-by: José Valim <jose.valim@dashbit.co>
This commit is contained in:
Jonatan Kłosko
GitHub
parent
8776ccdf31
commit
8802fd50f3
4 changed files with 30 additions and 218 deletions
|
|
@ -202,7 +202,7 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
|
|||
end
|
||||
|
||||
defp get_object(file_system, key) do
|
||||
case request(file_system, :get, "/" <> key) do
|
||||
case request(file_system, :get, "/" <> encode_key(key)) do
|
||||
{:ok, 200, _headers, body} -> {:ok, body}
|
||||
{:ok, 404, _headers, _body} -> FileSystem.Utils.posix_error(:enoent)
|
||||
other -> request_response_to_error(other)
|
||||
|
|
@ -210,14 +210,14 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
|
|||
end
|
||||
|
||||
defp put_object(file_system, key, content) do
|
||||
case request(file_system, :put, "/" <> key, body: content) |> encode() do
|
||||
case request(file_system, :put, "/" <> encode_key(key), body: content) |> encode() do
|
||||
{:ok, 200, _headers, _body} -> :ok
|
||||
other -> request_response_to_error(other)
|
||||
end
|
||||
end
|
||||
|
||||
defp head_object(file_system, key) do
|
||||
case request(file_system, :head, "/" <> key) do
|
||||
case request(file_system, :head, "/" <> encode_key(key)) do
|
||||
{:ok, 200, headers, _body} ->
|
||||
{:ok, etag} = HTTP.fetch_header(headers, "etag")
|
||||
{:ok, %{etag: etag}}
|
||||
|
|
@ -231,11 +231,12 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
|
|||
end
|
||||
|
||||
defp copy_object(file_system, bucket, source_key, destination_key) do
|
||||
copy_source = bucket <> "/" <> source_key
|
||||
copy_source = bucket <> "/" <> encode_key(source_key)
|
||||
|
||||
headers = [{"x-amz-copy-source", copy_source}]
|
||||
|
||||
case request(file_system, :put, "/" <> destination_key, headers: headers) |> encode() do
|
||||
case request(file_system, :put, "/" <> encode_key(destination_key), headers: headers)
|
||||
|> encode() do
|
||||
{:ok, 200, _headers, _body} -> :ok
|
||||
{:ok, 404, _headers, _body} -> FileSystem.Utils.posix_error(:enoent)
|
||||
other -> request_response_to_error(other)
|
||||
|
|
@ -243,7 +244,7 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
|
|||
end
|
||||
|
||||
defp delete_object(file_system, key) do
|
||||
case request(file_system, :delete, "/" <> key) |> encode() do
|
||||
case request(file_system, :delete, "/" <> encode_key(key)) |> encode() do
|
||||
{:ok, 204, _headers, _body} -> :ok
|
||||
{:ok, 404, _headers, _body} -> :ok
|
||||
other -> request_response_to_error(other)
|
||||
|
|
@ -287,6 +288,13 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
|
|||
end
|
||||
end
|
||||
|
||||
defp encode_key(key) do
|
||||
key
|
||||
|> String.split("/")
|
||||
|> Enum.map(fn segment -> URI.encode(segment, &URI.char_unreserved?/1) end)
|
||||
|> Enum.join("/")
|
||||
end
|
||||
|
||||
defp request_response_to_error(error)
|
||||
|
||||
defp request_response_to_error({:ok, 403, _headers, %{"Error" => %{"Message" => message}}}) do
|
||||
|
|
@ -326,19 +334,23 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
|
|||
|
||||
url = file_system.bucket_url <> path <> "?" <> URI.encode_query(query)
|
||||
|
||||
credentials = %{
|
||||
service: "s3",
|
||||
access_key_id: file_system.access_key_id,
|
||||
secret_access_key: file_system.secret_access_key,
|
||||
region: region_from_uri(file_system.bucket_url)
|
||||
}
|
||||
|
||||
now = NaiveDateTime.utc_now()
|
||||
now = NaiveDateTime.utc_now() |> NaiveDateTime.to_erl()
|
||||
|
||||
headers = [{"Host", host} | headers]
|
||||
|
||||
headers =
|
||||
Livebook.FileSystem.S3.Signature.sign_v4(credentials, now, method, url, headers, body || "")
|
||||
:aws_signature.sign_v4(
|
||||
file_system.access_key_id,
|
||||
file_system.secret_access_key,
|
||||
region_from_uri(file_system.bucket_url),
|
||||
"s3",
|
||||
now,
|
||||
Atom.to_string(method),
|
||||
url,
|
||||
headers,
|
||||
body || "",
|
||||
uri_encode_path: false
|
||||
)
|
||||
|
||||
body = body && {"application/octet-stream", body}
|
||||
HTTP.request(method, url, headers: headers, body: body)
|
||||
|
|
|
|||
|
|
@ -1,202 +0,0 @@
|
|||
defmodule Livebook.FileSystem.S3.Signature do
|
||||
@moduledoc false
|
||||
|
||||
# Adapted from https://github.com/aws-beam/aws-elixir/blob/v0.8.0/lib/aws/signature.ex
|
||||
#
|
||||
# Implements the Signature algorithm v4.
|
||||
# See: https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
|
||||
#
|
||||
# Briefly speaking, for every request we compute a signature
|
||||
# based on the auth credentials, headers, body and time. Upon
|
||||
# receiving the request, the S3 service computes the signature
|
||||
# on its own and compares with the one we sent.
|
||||
|
||||
@doc """
|
||||
Generate headers with an AWS signature version 4 for the specified
|
||||
request using the specified time.
|
||||
"""
|
||||
def sign_v4(credentials, now, method, url, headers, body) do
|
||||
now = NaiveDateTime.truncate(now, :second)
|
||||
long_date = NaiveDateTime.to_iso8601(now, :basic) <> "Z"
|
||||
short_date = Date.to_iso8601(now, :basic)
|
||||
|
||||
headers =
|
||||
headers
|
||||
|> add_date_header(long_date)
|
||||
|> add_content_hash(body)
|
||||
|
||||
canonical_request = canonical_request(method, url, headers, body)
|
||||
hashed_canonical_request = sha256_hexdigest(canonical_request)
|
||||
credential_scope = credential_scope(short_date, credentials.region, credentials.service)
|
||||
|
||||
signing_key = signing_key(credentials, short_date)
|
||||
|
||||
string_to_sign = string_to_sign(long_date, credential_scope, hashed_canonical_request)
|
||||
|
||||
signature = hmac_sha256_hexdigest(signing_key, string_to_sign)
|
||||
signed_headers = signed_headers(headers)
|
||||
|
||||
authorization =
|
||||
authorization(credentials.access_key_id, credential_scope, signed_headers, signature)
|
||||
|
||||
add_authorization_header(headers, authorization)
|
||||
end
|
||||
|
||||
defp add_date_header(headers, date) do
|
||||
[{"X-Amz-Date", date} | headers]
|
||||
end
|
||||
|
||||
# Add an X-Amz-Content-SHA256 header which is the hash of the payload.
|
||||
# This header is required for S3 when using the v4 signature. Adding it
|
||||
# in requests for all services does not cause any issues.
|
||||
defp add_content_hash(headers, body) do
|
||||
[{"X-Amz-Content-SHA256", sha256_hexdigest(body)} | headers]
|
||||
end
|
||||
|
||||
defp add_authorization_header(headers, authorization) do
|
||||
[{"Authorization", authorization} | headers]
|
||||
end
|
||||
|
||||
# Generate an AWS4-HMAC-SHA256 authorization signature.
|
||||
defp authorization(access_key_id, credential_scope, signed_headers, signature) do
|
||||
Enum.join(
|
||||
[
|
||||
"AWS4-HMAC-SHA256 ",
|
||||
"Credential=",
|
||||
access_key_id,
|
||||
"/",
|
||||
credential_scope,
|
||||
", ",
|
||||
"SignedHeaders=",
|
||||
signed_headers,
|
||||
", ",
|
||||
"Signature=",
|
||||
signature
|
||||
],
|
||||
""
|
||||
)
|
||||
end
|
||||
|
||||
# Convert a list of headers to canonical header format. Leading and trailing
|
||||
# whitespace around header names and values is stripped, header names are
|
||||
# lowercased, and headers are newline-joined in alphabetical order (with a
|
||||
# trailing newline).
|
||||
defp canonical_headers(headers) do
|
||||
headers
|
||||
|> Enum.map(fn {name, value} ->
|
||||
name = String.downcase(name, :ascii) |> String.trim()
|
||||
value = String.trim(value)
|
||||
{name, value}
|
||||
end)
|
||||
|> Enum.sort(fn {a, _}, {b, _} -> a <= b end)
|
||||
|> Enum.map(fn {name, value} -> [name, ":", value, "\n"] end)
|
||||
|> IO.iodata_to_binary()
|
||||
end
|
||||
|
||||
# Process and merge request values into a canonical request for AWS signature
|
||||
# version 4.
|
||||
defp canonical_request(method, url, headers, body) when is_atom(method) do
|
||||
Atom.to_string(method)
|
||||
|> String.upcase()
|
||||
|> canonical_request(url, headers, body)
|
||||
end
|
||||
|
||||
defp canonical_request(method, url, headers, body) do
|
||||
{canonical_url, canonical_query_string} = split_url(url)
|
||||
canonical_headers = canonical_headers(headers)
|
||||
signed_headers = signed_headers(headers)
|
||||
payload_hash = sha256_hexdigest(body)
|
||||
|
||||
Enum.join(
|
||||
[
|
||||
method,
|
||||
canonical_url,
|
||||
canonical_query_string,
|
||||
canonical_headers,
|
||||
signed_headers,
|
||||
payload_hash
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
end
|
||||
|
||||
# Generate a credential scope from a short date in `YYMMDD` format, a region
|
||||
# identifier and a service identifier.
|
||||
defp credential_scope(short_date, region, service) do
|
||||
Enum.join([short_date, region, service, "aws4_request"], "/")
|
||||
end
|
||||
|
||||
# Convert a list of headers to canonicals signed header format. Leading and
|
||||
# trailing whitespace around names is stripped, header names are lowercased,
|
||||
# and header names are semicolon-joined in alphabetical order.
|
||||
@spec signed_headers([{binary(), binary()}]) :: binary()
|
||||
defp signed_headers(headers) do
|
||||
headers
|
||||
|> Enum.map(fn {name, _value} -> name |> String.downcase() |> String.trim() end)
|
||||
|> Enum.sort()
|
||||
|> Enum.join(";")
|
||||
end
|
||||
|
||||
# Generate a signing key from a secret access key, a short date in `YYMMDD`
|
||||
# format, a region identifier and a service identifier.
|
||||
defp signing_key(%{} = credentials, short_date) do
|
||||
("AWS4" <> credentials.secret_access_key)
|
||||
|> hmac_sha256(short_date)
|
||||
|> hmac_sha256(credentials.region)
|
||||
|> hmac_sha256(credentials.service)
|
||||
|> hmac_sha256("aws4_request")
|
||||
end
|
||||
|
||||
# Strip the query string from the URL, if one if present, and return the URL
|
||||
# and the normalized query string as separate values.
|
||||
defp split_url(url) do
|
||||
url = URI.parse(url)
|
||||
|
||||
{uri_encode(url.path), normalize_query(url.query)}
|
||||
end
|
||||
|
||||
# Copied from https://github.com/ex-aws/ex_aws/blob/623478ed321ffc6c07fdd7236a2f0e03f1cbd517/lib/ex_aws/request/url.ex#L108
|
||||
defp uri_encode(url), do: URI.encode(url, &valid_path_char?/1)
|
||||
|
||||
defp valid_path_char?(?\s), do: false
|
||||
defp valid_path_char?(?/), do: true
|
||||
|
||||
defp valid_path_char?(c) do
|
||||
URI.char_unescaped?(c) && !URI.char_reserved?(c)
|
||||
end
|
||||
|
||||
# Sort query params by name first, then by value (if present). Append "=" to
|
||||
# params with missing value.
|
||||
# Example: "foo=bar&baz" becomes "baz=&foo=bar"
|
||||
defp normalize_query(nil), do: ""
|
||||
|
||||
defp normalize_query(query) do
|
||||
query
|
||||
|> URI.decode_query()
|
||||
|> Enum.sort()
|
||||
|> URI.encode_query()
|
||||
end
|
||||
|
||||
# Generate the text to sign from a long date in `YYMMDDTHHMMSSZ` format, a
|
||||
# credential scope and a hashed canonical request.
|
||||
defp string_to_sign(long_date, credential_scope, hashed_canonical_request) do
|
||||
Enum.join(
|
||||
["AWS4-HMAC-SHA256", long_date, credential_scope, hashed_canonical_request],
|
||||
"\n"
|
||||
)
|
||||
end
|
||||
|
||||
# Helpers
|
||||
|
||||
defp hmac_sha256(key, message) do
|
||||
:crypto.mac(:hmac, :sha256, key, message)
|
||||
end
|
||||
|
||||
defp hmac_sha256_hexdigest(key, message) do
|
||||
hmac_sha256(key, message) |> Base.encode16(case: :lower)
|
||||
end
|
||||
|
||||
defp sha256_hexdigest(value) do
|
||||
:crypto.hash(:sha256, value) |> Base.encode16(case: :lower)
|
||||
end
|
||||
end
|
||||
3
mix.exs
3
mix.exs
|
|
@ -53,7 +53,8 @@ defmodule Livebook.MixProject do
|
|||
{:plug_cowboy, "~> 2.0"},
|
||||
{:earmark_parser, "~> 1.4"},
|
||||
{:bypass, "~> 2.1", only: :test},
|
||||
{:castore, "~> 0.1.0"}
|
||||
{:castore, "~> 0.1.0"},
|
||||
{:aws_signature, "~> 0.1.0"}
|
||||
]
|
||||
end
|
||||
|
||||
|
|
|
|||
1
mix.lock
1
mix.lock
|
|
@ -1,4 +1,5 @@
|
|||
%{
|
||||
"aws_signature": {:hex, :aws_signature, "0.1.0", "d582c6c2d3f95a5bdd94436a983ffc94bd3686ba8f221b440e90d013829e6c5f", [:rebar3], [], "hexpm", "f645f1e1f3e56eca3d6b8e8efdeb060fc7f4bb1eace1d204fb3dcd9ab68998a4"},
|
||||
"bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
|
||||
"castore": {:hex, :castore, "0.1.11", "c0665858e0e1c3e8c27178e73dffea699a5b28eb72239a3b2642d208e8594914", [:mix], [], "hexpm", "91b009ba61973b532b84f7c09ce441cba7aa15cb8b006cf06c6f4bba18220081"},
|
||||
"cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue