diff --git a/lib/livebook_web/controllers/auth_controller.ex b/lib/livebook_web/controllers/auth_controller.ex
index 2829fd660..e06918725 100644
--- a/lib/livebook_web/controllers/auth_controller.ex
+++ b/lib/livebook_web/controllers/auth_controller.ex
@@ -25,7 +25,7 @@ defmodule LivebookWeb.AuthController do
     if AuthPlug.authenticated?(conn, :password) do
       redirect_to(conn)
     else
-      render_form_error(conn)
+      render_form_error(conn, :password)
     end
   end
 
@@ -35,14 +35,12 @@ defmodule LivebookWeb.AuthController do
     if AuthPlug.authenticated?(conn, :token) do
       redirect_to(conn)
     else
-      render_form_error(conn)
+      render_form_error(conn, :token)
     end
   end
 
-  defp render_form_error(conn) do
-    index(conn, %{
-      "errors" => [{"%{auth_mode} is invalid", [auth_mode: Livebook.Config.auth_mode()]}]
-    })
+  defp render_form_error(conn, auth_mode) do
+    index(conn, %{"errors" => [{"%{auth_mode} is invalid", [auth_mode: auth_mode]}]})
   end
 
   defp redirect_to(conn) do
diff --git a/lib/livebook_web/plugs/auth_plug.ex b/lib/livebook_web/plugs/auth_plug.ex
index 92e116018..1e2b5911b 100644
--- a/lib/livebook_web/plugs/auth_plug.ex
+++ b/lib/livebook_web/plugs/auth_plug.ex
@@ -51,7 +51,9 @@ defmodule LivebookWeb.AuthPlug do
 
   def authenticated?(session, port, mode) when mode in [:token, :password] do
     secret = session[key(port, mode)]
-    is_binary(secret) and Plug.Crypto.secure_compare(secret, expected(mode))
+
+    is_binary(secret) and mode == Livebook.Config.auth_mode() and
+      Plug.Crypto.secure_compare(secret, expected(mode))
   end
 
   defp authenticate(conn, :password) do
diff --git a/test/livebook_web/plugs/auth_plug_test.exs b/test/livebook_web/plugs/auth_plug_test.exs
index fb495f322..f68714fbc 100644
--- a/test/livebook_web/plugs/auth_plug_test.exs
+++ b/test/livebook_web/plugs/auth_plug_test.exs
@@ -2,14 +2,15 @@ defmodule LivebookWeb.AuthPlugTest do
   use LivebookWeb.ConnCase, async: false
 
   setup context do
-    {type, value} =
+    {type, other_type, value} =
       cond do
-        token = context[:token] -> {:token, token}
-        password = context[:password] -> {:password, password}
-        true -> {:disabled, ""}
+        token = context[:token] -> {:token, :password, token}
+        password = context[:password] -> {:password, :token, password}
+        true -> {:disabled, :disabled, ""}
       end
 
     unless type == :disabled do
+      Application.delete_env(:livebook, other_type)
       Application.put_env(:livebook, :authentication_mode, type)
       Application.put_env(:livebook, type, value)
 
@@ -100,6 +101,12 @@ defmodule LivebookWeb.AuthPlugTest do
       assert redirected_to(conn) == "/"
     end
 
+    @tag password: "grumpycat"
+    test "does not crash when given a token", %{conn: conn} do
+      conn = post(conn, "/authenticate?token=grumpycat")
+      assert html_response(conn, 200) =~ "token is invalid"
+    end
+
     @tag password: "grumpycat"
     test "redirects to '/authenticate' if not authenticated", %{conn: conn} do
       conn = get(conn, "/")