diff --git a/lib/livebook/application.ex b/lib/livebook/application.ex
index 17f31da62..7e5cde450 100644
--- a/lib/livebook/application.ex
+++ b/lib/livebook/application.ex
@@ -87,7 +87,7 @@ defmodule Livebook.Application do
 
   defp setup_optional_dependencies() do
     if Livebook.Config.aws_credentials?() do
-      Application.ensure_all_started(:aws_credentials)
+      {:ok, _} = Application.ensure_all_started(:aws_credentials)
     end
   end
 
diff --git a/mix.exs b/mix.exs
index 5425d715b..167a47531 100644
--- a/mix.exs
+++ b/mix.exs
@@ -153,15 +153,20 @@ defmodule Livebook.MixProject do
 
   ## Releases
 
+  # aws_credentials has runtime: false, so explicitly add is as :load
+  @release_apps [livebook: :permanent, aws_credentials: :load]
+
   defp releases do
     [
       livebook: [
+        applications: @release_apps,
         include_executables_for: [:unix, :windows],
         include_erts: false,
         rel_templates_path: "rel/server",
         steps: [:assemble, &remove_cookie/1]
       ],
       app: [
+        applications: @release_apps,
         include_erts: false,
         rel_templates_path: "rel/app",
         steps: [