diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1bbcb3366e..eb2278097d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -67,6 +67,10 @@ jobs:
       - run: make test-extensions
         name: Run In-tree Extension tests
 
+      - run: make ci-validate-dev
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'dependencies') }}
+        name: Validate dev mode will work
+
       - name: Install integration test dependencies
         id: minikube
         uses: medyagh/setup-minikube@5a9a7104d7322fa40424de8855c84685e89cefd7
diff --git a/Makefile b/Makefile
index 1435ab5004..6db96fbc2b 100644
--- a/Makefile
+++ b/Makefile
@@ -28,6 +28,9 @@ compile-dev: node_modules
 	yarn compile:main --cache
 	yarn compile:renderer --cache
 
+.PHONY: validate-dev
+ci-validate-dev: binaries/client build-extensions compile-dev
+
 .PHONY: dev
 dev: binaries/client build-extensions
 	rm -rf static/build/
diff --git a/package.json b/package.json
index cc11a7ef1c..01583e195f 100644
--- a/package.json
+++ b/package.json
@@ -278,7 +278,7 @@
     "@material-ui/core": "^4.12.3",
     "@material-ui/icons": "^4.11.2",
     "@material-ui/lab": "^4.0.0-alpha.60",
-    "@pmmmwh/react-refresh-webpack-plugin": "^0.5.4",
+    "@pmmmwh/react-refresh-webpack-plugin": "^0.5.5",
     "@sentry/types": "^6.19.3",
     "@testing-library/jest-dom": "^5.16.3",
     "@testing-library/react": "^12.1.4",
diff --git a/yarn.lock b/yarn.lock
index 916f262c6e..972b23276a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1003,10 +1003,10 @@
   resolved "https://registry.yarnpkg.com/@panva/asn1.js/-/asn1.js-1.0.0.tgz#dd55ae7b8129e02049f009408b97c61ccf9032f6"
   integrity sha512-UdkG3mLEqXgnlKsWanWcgb6dOjUzJ+XC5f+aWw30qrtjxeNUSfKX1cd5FBzOaXQumoe9nIqeZUvrRJS03HCCtw==
 
-"@pmmmwh/react-refresh-webpack-plugin@^0.5.4":
-  version "0.5.4"
-  resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.4.tgz#df0d0d855fc527db48aac93c218a0bf4ada41f99"
-  integrity sha512-zZbZeHQDnoTlt2AF+diQT0wsSXpvWiaIOZwBRdltNFhG1+I3ozyaw7U/nBiUwyJ0D+zwdXp0E3bWOl38Ag2BMw==
+"@pmmmwh/react-refresh-webpack-plugin@^0.5.5":
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.5.tgz#e77aac783bd079f548daa0a7f080ab5b5a9741ca"
+  integrity sha512-RbG7h6TuP6nFFYKJwbcToA1rjC1FyPg25NR2noAZ0vKI+la01KTSRPkuVPE+U88jXv7javx2JHglUcL1MHcshQ==
   dependencies:
     ansi-html-community "^0.0.8"
     common-path-prefix "^3.0.0"