diff --git a/.github/workflows/build-syncthing.yaml b/.github/workflows/build-syncthing.yaml
index 6bdf01957..f55e99212 100644
--- a/.github/workflows/build-syncthing.yaml
+++ b/.github/workflows/build-syncthing.yaml
@@ -49,6 +49,19 @@ jobs:
         # The oldest version in this list should match what we have in our go.mod.
         # Variables don't seem to be supported here, or we could have done something nice.
         go: ["1.20", "1.21"]
+
+        # Don't run the Windows tests with Go 1.21.4 or 1.20.11; this can be
+        # removed when 1.21.5 and 1.20.12 is released.
+        exclude:
+          - runner: windows-latest
+            go: "1.20"
+          - runner: windows-latest
+            go: "1.21"
+        include:
+          - runner: windows-latest
+            go: "~1.20.12 || ~1.20.0 <1.20.11"
+          - runner: windows-latest
+            go: "~1.21.5 || ~1.21.1 <1.21.4"
     runs-on: ${{ matrix.runner }}
     steps:
       - name: Set git to use LF
@@ -79,6 +92,7 @@ jobs:
 
       - name: Test
         run: |
+          go version
           go run build.go test | go-test-json-to-loki
         env:
           GOFLAGS: "-json"
@@ -155,7 +169,9 @@ jobs:
 
       - uses: actions/setup-go@v4
         with:
-          go-version: ${{ env.GO_VERSION }}
+          # Temporary version constraint to avoid 1.21.4 which has a bug in
+          # path handling. This can be removed when 1.21.5 is released.
+          go-version: "~1.21.5 || ~1.21.1 <1.21.4"
           cache: false
           check-latest: true