diff --git a/lua/harpoon/list.lua b/lua/harpoon/list.lua
index 8889177..425f863 100644
--- a/lua/harpoon/list.lua
+++ b/lua/harpoon/list.lua
@@ -14,10 +14,7 @@ local function guess_length(arr)
 end
 
 local function determine_length(arr, previous_length)
-    local idx = previous_length
-    if idx == 1 and arr[idx] == nil then
-        return 0
-    end 
+    local idx = 0
     for i = previous_length, 1, -1 do
         if arr[i] ~= nil then
             idx = i
diff --git a/lua/harpoon/test/list_spec.lua b/lua/harpoon/test/list_spec.lua
index 7adbbc3..6a3bd10 100644
--- a/lua/harpoon/test/list_spec.lua
+++ b/lua/harpoon/test/list_spec.lua
@@ -193,6 +193,16 @@ describe("list", function()
         }, list.items)
     end)
 
+    it("remove #653 bug", function()
+        local config = Config.merge_config()
+        local c = Config.get_config(config, "foo")
+        local list = List:new(c, "foo", {
+            { value = "one" },
+        })
+        list:remove_at(1)
+        eq(0, list:length())
+    end)
+
     it("remove_at", function()
         local config = Config.merge_config()
         local c = Config.get_config(config, "foo")