diff --git a/.github/workflows/rustc-pull.yml b/.github/workflows/rustc-pull.yml
index 2a842f3b31..37cf5f3726 100644
--- a/.github/workflows/rustc-pull.yml
+++ b/.github/workflows/rustc-pull.yml
@@ -11,10 +11,11 @@ jobs:
     if: github.repository == 'rust-lang/rust-analyzer'
     uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@main
     with:
+      github-app-id: ${{ vars.APP_CLIENT_ID }}
       zulip-stream-id: 185405
       zulip-bot-email:  "rust-analyzer-ci-bot@rust-lang.zulipchat.com"
       pr-base-branch: master
       branch-name: rustc-pull
     secrets:
       zulip-api-token: ${{ secrets.ZULIP_API_TOKEN }}
-      token: ${{ secrets.GITHUB_TOKEN }}
+      github-app-secret: ${{ secrets.APP_PRIVATE_KEY }}
diff --git a/triagebot.toml b/triagebot.toml
index 27fdb67245..c9862495bc 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -28,6 +28,3 @@ labels = ["has-merge-commits", "S-waiting-on-author"]
 
 # Prevents mentions in commits to avoid users being spammed
 [no-mentions]
-
-# Automatically close and reopen PRs made by bots to run CI on them
-[bot-pull-requests]