diff --git a/README.md b/README.md
index 250d21c..3bee187 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ jobs:
       - name: Install Dependencies
         run: yarn
 
-      - name: Create Pull Request or Publish to npm
+      - name: Create Release Pull Request
         uses: changesets/action@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -72,7 +72,7 @@ jobs:
       - name: Install Dependencies
         run: yarn
 
-      - name: "Create Pull Request or Publish to npm"
+      - name: Create Release Pull Request or Publish to npm
         uses: changesets/action@master
         with:
           # The --otp=1 is to get around a bug in changesets
diff --git a/dist/index.js b/dist/index.js
index 6bcd550..396d6c2 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -985,5 +985,5 @@ function e(e){return Array.isArray(e)?e:[e]}const t=/^\s+$/,r=/^\\!/,i=/^\\#/,s=
 },{"array-union":"iMRL","glob":"R9Ja","fast-glob":"qwKf","dir-glob":"MgqK","./gitignore":"Rx5v"}],"kAcF":[function(require,module,exports) {
 "use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var e=t(require("fs-extra")),a=t(require("path")),r=t(require("globby"));function t(e){return e&&e.__esModule?e:{default:e}}async function s(t={}){const s=t.cwd||process.cwd(),o=t.tools||["yarn","bolt"],n=await e.default.readFile(a.default.join(s,"package.json"),"utf-8").then(JSON.parse);let l;if(o.includes("yarn")&&n.workspaces?Array.isArray(n.workspaces)?l=n.workspaces:n.workspaces.packages&&(l=n.workspaces.packages):o.includes("bolt")&&n.bolt&&n.bolt.workspaces&&(l=n.bolt.workspaces),!l)return o.includes("root")?[{config:n,dir:s,name:n.name}]:null;const i=await(0,r.default)(l,{cwd:s,onlyDirectories:!0,absolute:!0,expandDirectories:!1});let c=[];const u=await Promise.all(i.sort().filter(r=>e.default.existsSync(a.default.join(r,"package.json"))).map(async r=>e.default.readFile(a.default.join(r,"package.json"),"utf8").then(e=>{const t=JSON.parse(e);return t.name||c.push(a.default.relative(s,a.default.join(r,"package.json"))),{config:t,name:t.name,dir:r}})));if(0!==c.length)throw c.sort(),new Error(`The following package.jsons are missing the "name" field:\n${c.join("\n")}`);return u}var o=s;exports.default=o;
 },{"fs-extra":"wFa+","globby":"e8e8"}],"7QCb":[function(require,module,exports) {
-"use strict";function e(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter(function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable})),n.push.apply(n,a)}return n}function t(t){for(var a=1;a<arguments.length;a++){var r=null!=arguments[a]?arguments[a]:{};a%2?e(r,!0).forEach(function(e){n(t,e,r[e])}):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(r)):e(r).forEach(function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(r,e))})}return t}function n(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}var a=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t},r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0});const o=a(require("@actions/core")),i=require("@actions/exec"),s=a(require("@actions/github")),l=r(require("fs-extra")),c=r(require("unified")),u=r(require("remark-parse")),d=r(require("remark-stringify")),g=r(require("mdast-util-to-string")),f=r(require("get-workspaces")),h=r(require("path"));async function p(e,n,a){let r="",o="";return{code:await i.exec(e,n,t({listeners:{stdout:e=>{r+=e.toString()},stderr:e=>{o+=e.toString()}}},a)),stdout:r,stderr:o}}function b(e,t){let n,a,r=c.default().use(u.default).parse(e),o=r.children;for(let i=0;i<o.length;i++){let e=o[i];if(void 0!==n||"heading"!==e.type||g.default(e)!==t){if(void 0!==n&&"heading"===e.type&&n.depth===e.depth){a=i;break}}else n={index:i,depth:e.depth}}return n&&(r.children=r.children.slice(n.index+1,a)),c.default().use(d.default).stringify(r)}(async()=>{let e=process.env.GITHUB_TOKEN;if(!e)return void o.setFailed("Please add the GITHUB_TOKEN to the changesets action");let n=`${s.context.repo.owner}/${s.context.repo.repo}`;const a=new s.GitHub(e);let r=a.repos.get(s.context.repo).then(e=>e.data.default_branch);console.log("setting git user"),await i.exec("git",["config","--global","user.name",'"github-actions[bot]"']),await i.exec("git",["config","--global","user.email",'"github-actions[bot]@users.noreply.github.com"']),console.log("setting GitHub credentials"),await l.default.writeFile(`${process.env.HOME}/.netrc`,`machine github.com\nlogin github-actions[bot]\npassword ${e}`);let c=await r;if(s.context.ref.replace("refs/heads/","")!==c)return void o.setFailed(`The changesets action should only run on ${c} but it's running on ${s.context.ref.replace("refs/heads/","")}, please change your GitHub actions config to only run the Changesets action on ${c}`);let u=l.default.readdirSync(`${process.cwd()}/.changeset`).some(e=>"config.js"!==e&&"README.md"!==e),d=o.getInput("publish");if(!u&&!d)return void console.log("No changesets found");if(!u&&d){console.log("No changesets found, attempting to publish any unpublished packages to npm");let e=await f.default({tools:["bolt","yarn","root"]});if(!e)return o.setFailed("Could not find workspaces");let n=new Map(e.map(e=>[e.name,e]));l.default.writeFileSync(`${process.env.HOME}/.npmrc`,`//registry.npmjs.org/:_authToken=${process.env.NPM_TOKEN}`);let[r,...u]=d.split(/\s+/),g=await p(r,u);await i.exec("git",["push","origin",`HEAD:${c}`,"--follow-tags"]);let w=/New tag:\s+(@[^\/]+\/[^@]+|[^\/]+)@([^\s]+)/,m=[];for(let t of g.stdout.split("\n")){let e=t.match(w);if(null===e)continue;let a=e[1],r=n.get(a);if(void 0===r)return o.setFailed("Workspace not found for "+a+". This is probably a bug in the action, please open an issue");m.push(r)}return void(await Promise.all(m.map(async e=>{try{let r=h.default.join(e.dir,"CHANGELOG.md"),i=b(await l.default.readFile(r,"utf8"),e.config.version);if(!i)return o.setFailed(`Could not find changelog entry for ${e.name}@${e.config.version}`);await a.repos.createRelease(t({tag_name:`${e.name}@${e.config.version}`,body:i,prerelease:e.config.version.includes("-")},s.context.repo))}catch(n){if("ENOENT"!==n.code)throw n}})))}let{stderr:g}=await p("git",["checkout","changeset-release"],{ignoreReturnCode:!0}),w=!g.toString().includes("Switched to a new branch 'changeset-release'");w&&await i.exec("git",["checkout","-b","changeset-release"]);let m=w;if(!m){console.log("checking if new changesets should be added");const e=(await p("git",["merge-base","changeset-release",s.context.sha])).stdout.trim();m=(await p("git",["diff","--name-only",`${e}...${s.context.sha}`])).stdout.trim().includes(".changeset"),console.log("checked if new changesets should be added "+m)}if(m){await i.exec("git",["reset","--hard",s.context.sha]);let e=await p("yarn",["changeset","bump"]),r=`repo:${n}+state:open+head:changeset-release+base:${c}`,o=a.search.issuesAndPullRequests({q:r}),u=(async()=>{let t=/Updated file (.+)/,n=[];for(let a of e.stdout.split("\n")){let e=a.match(t);null!==e&&n.push(e[1])}return"Hey!!\n\nThis PR was automatically opened by the Changesets release GitHub action. When you're ready to do a release, you can merge this and publish to npm yourself or \n\n# Changes\n"+(await Promise.all(n.map(async e=>{let[t,n]=await Promise.all([l.default.readJson(h.default.join(h.default.dirname(e),"package.json")),l.default.readFile(e,"utf8")]),a=b(n,t.version);return`## ${t.name}@${t.version}\n`+a}))).join("\n ")})();await i.exec("git",["add","."]),await i.exec("git",["commit","-m","Version Packages"]),await i.exec("git",["push","origin","changeset-release","--force"]);let d=await o;console.log(JSON.stringify(d.data,null,2)),0===d.data.items.length?(console.log("creating pull request"),await a.pulls.create(t({base:c,head:"changeset-release",title:"Version Packages",body:await u},s.context.repo))):(a.pulls.update(t({pull_number:d.data.items[0].number,body:await u},s.context.repo)),console.log("pull request found"))}else console.log("no new changesets")})().catch(e=>{console.error(e),o.setFailed(e.message)});
+"use strict";function e(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter(function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable})),a.push.apply(a,n)}return a}function t(t){for(var n=1;n<arguments.length;n++){var o=null!=arguments[n]?arguments[n]:{};n%2?e(o,!0).forEach(function(e){a(t,e,o[e])}):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(o)):e(o).forEach(function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(o,e))})}return t}function a(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}var n=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var a in e)Object.hasOwnProperty.call(e,a)&&(t[a]=e[a]);return t.default=e,t},o=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0});const r=n(require("@actions/core")),i=require("@actions/exec"),s=n(require("@actions/github")),l=o(require("fs-extra")),c=o(require("unified")),u=o(require("remark-parse")),d=o(require("remark-stringify")),g=o(require("mdast-util-to-string")),h=o(require("get-workspaces")),p=o(require("path"));async function f(e,a,n){let o="",r="";return{code:await i.exec(e,a,t({listeners:{stdout:e=>{o+=e.toString()},stderr:e=>{r+=e.toString()}}},n)),stdout:o,stderr:r}}function b(e,t){let a,n,o=c.default().use(u.default).parse(e),r=o.children;for(let i=0;i<r.length;i++){let e=r[i];if(void 0!==a||"heading"!==e.type||g.default(e)!==t){if(void 0!==a&&"heading"===e.type&&a.depth===e.depth){n=i;break}}else a={index:i,depth:e.depth}}return a&&(o.children=o.children.slice(a.index+1,n)),c.default().use(d.default).stringify(o)}(async()=>{let e=process.env.GITHUB_TOKEN;if(!e)return void r.setFailed("Please add the GITHUB_TOKEN to the changesets action");let a=`${s.context.repo.owner}/${s.context.repo.repo}`;const n=new s.GitHub(e);let o=n.repos.get(s.context.repo).then(e=>e.data.default_branch);console.log("setting git user"),await i.exec("git",["config","--global","user.name",'"github-actions[bot]"']),await i.exec("git",["config","--global","user.email",'"github-actions[bot]@users.noreply.github.com"']),console.log("setting GitHub credentials"),await l.default.writeFile(`${process.env.HOME}/.netrc`,`machine github.com\nlogin github-actions[bot]\npassword ${e}`);let c=await o;if(s.context.ref.replace("refs/heads/","")!==c)return void r.setFailed(`The changesets action should only run on ${c} but it's running on ${s.context.ref.replace("refs/heads/","")}, please change your GitHub actions config to only run the Changesets action on ${c}`);let u=l.default.readdirSync(`${process.cwd()}/.changeset`).some(e=>"config.js"!==e&&"README.md"!==e),d=r.getInput("publish");if(!u&&!d)return void console.log("No changesets found");if(!u&&d){console.log("No changesets found, attempting to publish any unpublished packages to npm");let e=await h.default({tools:["bolt","yarn","root"]});if(!e)return r.setFailed("Could not find workspaces");let a=new Map(e.map(e=>[e.name,e]));l.default.writeFileSync(`${process.env.HOME}/.npmrc`,`//registry.npmjs.org/:_authToken=${process.env.NPM_TOKEN}`);let[o,...u]=d.split(/\s+/),g=await f(o,u);await i.exec("git",["push","origin",`HEAD:${c}`,"--follow-tags"]);let w=/New tag:\s+(@[^\/]+\/[^@]+|[^\/]+)@([^\s]+)/,m=[];for(let t of g.stdout.split("\n")){let e=t.match(w);if(null===e)continue;let n=e[1],o=a.get(n);if(void 0===o)return r.setFailed("Workspace not found for "+n+". This is probably a bug in the action, please open an issue");m.push(o)}return void(await Promise.all(m.map(async e=>{try{let o=p.default.join(e.dir,"CHANGELOG.md"),i=b(await l.default.readFile(o,"utf8"),e.config.version);if(!i)return r.setFailed(`Could not find changelog entry for ${e.name}@${e.config.version}`);await n.repos.createRelease(t({tag_name:`${e.name}@${e.config.version}`,body:i,prerelease:e.config.version.includes("-")},s.context.repo))}catch(a){if("ENOENT"!==a.code)throw a}})))}let{stderr:g}=await f("git",["checkout","changeset-release"],{ignoreReturnCode:!0}),w=!g.toString().includes("Switched to a new branch 'changeset-release'");w&&await i.exec("git",["checkout","-b","changeset-release"]);let m=w;if(!m){console.log("checking if new changesets should be added");const e=(await f("git",["merge-base","changeset-release",s.context.sha])).stdout.trim();m=(await f("git",["diff","--name-only",`${e}...${s.context.sha}`])).stdout.trim().includes(".changeset"),console.log("checked if new changesets should be added "+m)}if(m){await i.exec("git",["reset","--hard",s.context.sha]);let e=await f("yarn",["changeset","bump"]),o=`repo:${a}+state:open+head:changeset-release+base:${c}`,r=n.search.issuesAndPullRequests({q:o}),u=(async()=>{let t=/Updated file (.+)/,a=[];for(let n of e.stdout.split("\n")){let e=n.match(t);null!==e&&a.push(e[1])}return`Hey!! 🦋\n\nThis PR was opened by the Changesets release GitHub action. When you're ready to do a release, you can merge this and ${d?"the packages will be published to npm automatically":"publish to npm yourself or [setup this action to publish automatically](https://github.com/changesets/action#with-publishing)"}. If you're not ready to do a release yet, that's fine, whenever you add more changesets to ${c}, this PR will be updated.\n\n# Releases\n`+(await Promise.all(a.map(async e=>{let[t,a]=await Promise.all([l.default.readJson(p.default.join(p.default.dirname(e),"package.json")),l.default.readFile(e,"utf8")]),n=b(a,t.version);return`## ${t.name}@${t.version}\n\n`+n}))).join("\n ")})();await i.exec("git",["add","."]),await i.exec("git",["commit","-m","Version Packages"]),await i.exec("git",["push","origin","changeset-release","--force"]);let g=await r;console.log(JSON.stringify(g.data,null,2)),0===g.data.items.length?(console.log("creating pull request"),await n.pulls.create(t({base:c,head:"changeset-release",title:"Version Packages",body:await u},s.context.repo))):(n.pulls.update(t({pull_number:g.data.items[0].number,body:await u},s.context.repo)),console.log("pull request found"))}else console.log("no new changesets")})().catch(e=>{console.error(e),r.setFailed(e.message)});
 },{"@actions/core":"4FTV","@actions/exec":"utFe","@actions/github":"8Dol","fs-extra":"JTJ9","unified":"EJQW","remark-parse":"+8Y3","remark-stringify":"eCnT","mdast-util-to-string":"ljHw","get-workspaces":"kAcF"}]},{},["7QCb"], null)
\ No newline at end of file
diff --git a/index.ts b/index.ts
index 3442af6..66aa064 100644
--- a/index.ts
+++ b/index.ts
@@ -213,14 +213,54 @@ async function execWithOutput(
   }
   if (shouldBump) {
     await exec("git", ["reset", "--hard", github.context.sha]);
-    await exec("yarn", ["changeset", "bump"]);
+    let output = await execWithOutput("yarn", ["changeset", "bump"]);
+    let searchQuery = `repo:${repo}+state:open+head:changeset-release+base:${defaultBranch}`;
+    let searchResultPromise = octokit.search.issuesAndPullRequests({
+      q: searchQuery
+    });
+    let prBodyPromise = (async () => {
+      let updateChangelogRegex = /Updated file (.+)/;
+      let changelogFilenames = [];
+      for (let line of output.stdout.split("\n")) {
+        let match = line.match(updateChangelogRegex);
+        if (match === null) {
+          continue;
+        }
+        changelogFilenames.push(match[1]);
+      }
+      return (
+        `Hey!! 🦋
+
+This PR was opened by the Changesets release GitHub action. When you're ready to do a release, you can merge this and ${
+          publishScript
+            ? `the packages will be published to npm automatically`
+            : `publish to npm yourself or [setup this action to publish automatically](https://github.com/changesets/action#with-publishing)`
+        }. If you're not ready to do a release yet, that's fine, whenever you add more changesets to ${defaultBranch}, this PR will be updated.
+
+# Releases
+` +
+        (await Promise.all(
+          changelogFilenames.map(async filename => {
+            let [pkgJsonContent, changelogContents] = await Promise.all([
+              fs.readJson(path.join(path.dirname(filename), "package.json")),
+              fs.readFile(filename, "utf8")
+            ]);
+
+            let entry = getChangelogEntry(
+              changelogContents,
+              pkgJsonContent.version
+            );
+            return (
+              `## ${pkgJsonContent.name}@${pkgJsonContent.version}\n\n` + entry
+            );
+          })
+        )).join("\n ")
+      );
+    })();
     await exec("git", ["add", "."]);
     await exec("git", ["commit", "-m", "Version Packages"]);
     await exec("git", ["push", "origin", "changeset-release", "--force"]);
-    let searchQuery = `repo:${repo}+state:open+head:changeset-release+base:${defaultBranch}`;
-    let searchResult = await octokit.search.issuesAndPullRequests({
-      q: searchQuery
-    });
+    let searchResult = await searchResultPromise;
     console.log(JSON.stringify(searchResult.data, null, 2));
     if (searchResult.data.items.length === 0) {
       console.log("creating pull request");
@@ -228,9 +268,15 @@ async function execWithOutput(
         base: defaultBranch,
         head: "changeset-release",
         title: "Version Packages",
+        body: await prBodyPromise,
         ...github.context.repo
       });
     } else {
+      octokit.pulls.update({
+        pull_number: searchResult.data.items[0].number,
+        body: await prBodyPromise,
+        ...github.context.repo
+      });
       console.log("pull request found");
     }
   } else {