Sign S3 requests using aws_signature (#497)

* Sign S3 requests using aws_signature

* escape_key -> encode_key

* Update lib/livebook/file_system/s3.ex

Co-authored-by: José Valim <jose.valim@dashbit.co>

Co-authored-by: José Valim <jose.valim@dashbit.co>

lib/livebook/file_system/s3.ex

@@ -202,7 +202,7 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
   end
 
   defp get_object(file_system, key) do
-    case request(file_system, :get, "/" <> key) do
+    case request(file_system, :get, "/" <> encode_key(key)) do
       {:ok, 200, _headers, body} -> {:ok, body}
       {:ok, 404, _headers, _body} -> FileSystem.Utils.posix_error(:enoent)
       other -> request_response_to_error(other)
@@ -210,14 +210,14 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
   end
 
   defp put_object(file_system, key, content) do
-    case request(file_system, :put, "/" <> key, body: content) |> encode() do
+    case request(file_system, :put, "/" <> encode_key(key), body: content) |> encode() do
       {:ok, 200, _headers, _body} -> :ok
       other -> request_response_to_error(other)
     end
   end
 
   defp head_object(file_system, key) do
-    case request(file_system, :head, "/" <> key) do
+    case request(file_system, :head, "/" <> encode_key(key)) do
       {:ok, 200, headers, _body} ->
         {:ok, etag} = HTTP.fetch_header(headers, "etag")
         {:ok, %{etag: etag}}
@@ -231,11 +231,12 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
   end
 
   defp copy_object(file_system, bucket, source_key, destination_key) do
-    copy_source = bucket <> "/" <> source_key
+    copy_source = bucket <> "/" <> encode_key(source_key)
 
     headers = [{"x-amz-copy-source", copy_source}]
 
-    case request(file_system, :put, "/" <> destination_key, headers: headers) |> encode() do
+    case request(file_system, :put, "/" <> encode_key(destination_key), headers: headers)
+         |> encode() do
       {:ok, 200, _headers, _body} -> :ok
       {:ok, 404, _headers, _body} -> FileSystem.Utils.posix_error(:enoent)
       other -> request_response_to_error(other)
@@ -243,7 +244,7 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
   end
 
   defp delete_object(file_system, key) do
-    case request(file_system, :delete, "/" <> key) |> encode() do
+    case request(file_system, :delete, "/" <> encode_key(key)) |> encode() do
       {:ok, 204, _headers, _body} -> :ok
       {:ok, 404, _headers, _body} -> :ok
       other -> request_response_to_error(other)
@@ -287,6 +288,13 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
     end
   end
 
+  defp encode_key(key) do
+    key
+    |> String.split("/")
+    |> Enum.map(fn segment -> URI.encode(segment, &URI.char_unreserved?/1) end)
+    |> Enum.join("/")
+  end
+
   defp request_response_to_error(error)
 
   defp request_response_to_error({:ok, 403, _headers, %{"Error" => %{"Message" => message}}}) do
@@ -326,19 +334,23 @@ defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do
 
     url = file_system.bucket_url <> path <> "?" <> URI.encode_query(query)
 
-    credentials = %{
+    now = NaiveDateTime.utc_now() |> NaiveDateTime.to_erl()
-      service: "s3",
-      access_key_id: file_system.access_key_id,
-      secret_access_key: file_system.secret_access_key,
-      region: region_from_uri(file_system.bucket_url)
-    }
-
-    now = NaiveDateTime.utc_now()
 
     headers = [{"Host", host} | headers]
 
     headers =
-      Livebook.FileSystem.S3.Signature.sign_v4(credentials, now, method, url, headers, body || "")
+      :aws_signature.sign_v4(
+        file_system.access_key_id,
+        file_system.secret_access_key,
+        region_from_uri(file_system.bucket_url),
+        "s3",
+        now,
+        Atom.to_string(method),
+        url,
+        headers,
+        body || "",
+        uri_encode_path: false
+      )
 
     body = body && {"application/octet-stream", body}
     HTTP.request(method, url, headers: headers, body: body)

lib/livebook/file_system/s3/signature.ex

@@ -1,202 +0,0 @@
-defmodule Livebook.FileSystem.S3.Signature do
-  @moduledoc false
-
-  # Adapted from https://github.com/aws-beam/aws-elixir/blob/v0.8.0/lib/aws/signature.ex
-  #
-  # Implements the Signature algorithm v4.
-  # See: https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
-  #
-  # Briefly speaking, for every request we compute a signature
-  # based on the auth credentials, headers, body and time. Upon
-  # receiving the request, the S3 service computes the signature
-  # on its own and compares with the one we sent.
-
-  @doc """
-  Generate headers with an AWS signature version 4 for the specified
-  request using the specified time.
-  """
-  def sign_v4(credentials, now, method, url, headers, body) do
-    now = NaiveDateTime.truncate(now, :second)
-    long_date = NaiveDateTime.to_iso8601(now, :basic) <> "Z"
-    short_date = Date.to_iso8601(now, :basic)
-
-    headers =
-      headers
-      |> add_date_header(long_date)
-      |> add_content_hash(body)
-
-    canonical_request = canonical_request(method, url, headers, body)
-    hashed_canonical_request = sha256_hexdigest(canonical_request)
-    credential_scope = credential_scope(short_date, credentials.region, credentials.service)
-
-    signing_key = signing_key(credentials, short_date)
-
-    string_to_sign = string_to_sign(long_date, credential_scope, hashed_canonical_request)
-
-    signature = hmac_sha256_hexdigest(signing_key, string_to_sign)
-    signed_headers = signed_headers(headers)
-
-    authorization =
-      authorization(credentials.access_key_id, credential_scope, signed_headers, signature)
-
-    add_authorization_header(headers, authorization)
-  end
-
-  defp add_date_header(headers, date) do
-    [{"X-Amz-Date", date} | headers]
-  end
-
-  # Add an X-Amz-Content-SHA256 header which is the hash of the payload.
-  # This header is required for S3 when using the v4 signature. Adding it
-  # in requests for all services does not cause any issues.
-  defp add_content_hash(headers, body) do
-    [{"X-Amz-Content-SHA256", sha256_hexdigest(body)} | headers]
-  end
-
-  defp add_authorization_header(headers, authorization) do
-    [{"Authorization", authorization} | headers]
-  end
-
-  # Generate an AWS4-HMAC-SHA256 authorization signature.
-  defp authorization(access_key_id, credential_scope, signed_headers, signature) do
-    Enum.join(
-      [
-        "AWS4-HMAC-SHA256 ",
-        "Credential=",
-        access_key_id,
-        "/",
-        credential_scope,
-        ", ",
-        "SignedHeaders=",
-        signed_headers,
-        ", ",
-        "Signature=",
-        signature
-      ],
-      ""
-    )
-  end
-
-  # Convert a list of headers to canonical header format.  Leading and trailing
-  # whitespace around header names and values is stripped, header names are
-  # lowercased, and headers are newline-joined in alphabetical order (with a
-  # trailing newline).
-  defp canonical_headers(headers) do
-    headers
-    |> Enum.map(fn {name, value} ->
-      name = String.downcase(name, :ascii) |> String.trim()
-      value = String.trim(value)
-      {name, value}
-    end)
-    |> Enum.sort(fn {a, _}, {b, _} -> a <= b end)
-    |> Enum.map(fn {name, value} -> [name, ":", value, "\n"] end)
-    |> IO.iodata_to_binary()
-  end
-
-  # Process and merge request values into a canonical request for AWS signature
-  # version 4.
-  defp canonical_request(method, url, headers, body) when is_atom(method) do
-    Atom.to_string(method)
-    |> String.upcase()
-    |> canonical_request(url, headers, body)
-  end
-
-  defp canonical_request(method, url, headers, body) do
-    {canonical_url, canonical_query_string} = split_url(url)
-    canonical_headers = canonical_headers(headers)
-    signed_headers = signed_headers(headers)
-    payload_hash = sha256_hexdigest(body)
-
-    Enum.join(
-      [
-        method,
-        canonical_url,
-        canonical_query_string,
-        canonical_headers,
-        signed_headers,
-        payload_hash
-      ],
-      "\n"
-    )
-  end
-
-  # Generate a credential scope from a short date in `YYMMDD` format, a region
-  # identifier and a service identifier.
-  defp credential_scope(short_date, region, service) do
-    Enum.join([short_date, region, service, "aws4_request"], "/")
-  end
-
-  # Convert a list of headers to canonicals signed header format.  Leading and
-  # trailing whitespace around names is stripped, header names are lowercased,
-  # and header names are semicolon-joined in alphabetical order.
-  @spec signed_headers([{binary(), binary()}]) :: binary()
-  defp signed_headers(headers) do
-    headers
-    |> Enum.map(fn {name, _value} -> name |> String.downcase() |> String.trim() end)
-    |> Enum.sort()
-    |> Enum.join(";")
-  end
-
-  # Generate a signing key from a secret access key, a short date in `YYMMDD`
-  # format, a region identifier and a service identifier.
-  defp signing_key(%{} = credentials, short_date) do
-    ("AWS4" <> credentials.secret_access_key)
-    |> hmac_sha256(short_date)
-    |> hmac_sha256(credentials.region)
-    |> hmac_sha256(credentials.service)
-    |> hmac_sha256("aws4_request")
-  end
-
-  # Strip the query string from the URL, if one if present, and return the URL
-  # and the normalized query string as separate values.
-  defp split_url(url) do
-    url = URI.parse(url)
-
-    {uri_encode(url.path), normalize_query(url.query)}
-  end
-
-  # Copied from https://github.com/ex-aws/ex_aws/blob/623478ed321ffc6c07fdd7236a2f0e03f1cbd517/lib/ex_aws/request/url.ex#L108
-  defp uri_encode(url), do: URI.encode(url, &valid_path_char?/1)
-
-  defp valid_path_char?(?\s), do: false
-  defp valid_path_char?(?/), do: true
-
-  defp valid_path_char?(c) do
-    URI.char_unescaped?(c) && !URI.char_reserved?(c)
-  end
-
-  # Sort query params by name first, then by value (if present). Append "=" to
-  # params with missing value.
-  # Example: "foo=bar&baz" becomes "baz=&foo=bar"
-  defp normalize_query(nil), do: ""
-
-  defp normalize_query(query) do
-    query
-    |> URI.decode_query()
-    |> Enum.sort()
-    |> URI.encode_query()
-  end
-
-  # Generate the text to sign from a long date in `YYMMDDTHHMMSSZ` format, a
-  # credential scope and a hashed canonical request.
-  defp string_to_sign(long_date, credential_scope, hashed_canonical_request) do
-    Enum.join(
-      ["AWS4-HMAC-SHA256", long_date, credential_scope, hashed_canonical_request],
-      "\n"
-    )
-  end
-
-  # Helpers
-
-  defp hmac_sha256(key, message) do
-    :crypto.mac(:hmac, :sha256, key, message)
-  end
-
-  defp hmac_sha256_hexdigest(key, message) do
-    hmac_sha256(key, message) |> Base.encode16(case: :lower)
-  end
-
-  defp sha256_hexdigest(value) do
-    :crypto.hash(:sha256, value) |> Base.encode16(case: :lower)
-  end
-end

mix.exs

@@ -53,7 +53,8 @@ defmodule Livebook.MixProject do
       {:plug_cowboy, "~> 2.0"},
       {:earmark_parser, "~> 1.4"},
       {:bypass, "~> 2.1", only: :test},
-      {:castore, "~> 0.1.0"}
+      {:castore, "~> 0.1.0"},
+      {:aws_signature, "~> 0.1.0"}
     ]
   end
 

mix.lock

@@ -1,4 +1,5 @@
 %{
+  "aws_signature": {:hex, :aws_signature, "0.1.0", "d582c6c2d3f95a5bdd94436a983ffc94bd3686ba8f221b440e90d013829e6c5f", [:rebar3], [], "hexpm", "f645f1e1f3e56eca3d6b8e8efdeb060fc7f4bb1eace1d204fb3dcd9ab68998a4"},
   "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
   "castore": {:hex, :castore, "0.1.11", "c0665858e0e1c3e8c27178e73dffea699a5b28eb72239a3b2642d208e8594914", [:mix], [], "hexpm", "91b009ba61973b532b84f7c09ce441cba7aa15cb8b006cf06c6f4bba18220081"},
   "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},