diff --git a/windows/winpgen.c b/windows/winpgen.c
index ffbeb1a4..aeebb629 100644
--- a/windows/winpgen.c
+++ b/windows/winpgen.c
@@ -41,6 +41,22 @@ void modalfatalbox(char *fmt, ...)
     exit(1);
 }
 
+/*
+ * Print a non-fatal message box and do not exit.
+ */
+void nonfatal(char *fmt, ...)
+{
+    va_list ap;
+    char *stuff;
+
+    va_start(ap, fmt);
+    stuff = dupvprintf(fmt, ap);
+    va_end(ap);
+    MessageBox(NULL, stuff, "PuTTYgen Error",
+	       MB_SYSTEMMODAL | MB_ICONERROR | MB_OK);
+    sfree(stuff);
+}
+
 /* ----------------------------------------------------------------------
  * Progress report code. This is really horrible :-)
  */
diff --git a/windows/winstore.c b/windows/winstore.c
index 76c73d09..66c7c759 100644
--- a/windows/winstore.c
+++ b/windows/winstore.c
@@ -456,7 +456,10 @@ enum { DEL, OPEN_R, OPEN_W };
 static int try_random_seed(char const *path, int action, HANDLE *ret)
 {
     if (action == DEL) {
-	remove(path);
+        if (!DeleteFile(path) && GetLastError() != ERROR_FILE_NOT_FOUND) {
+            nonfatal("Unable to delete '%s': %s", path,
+                     win_strerror(GetLastError()));
+        }
 	*ret = INVALID_HANDLE_VALUE;
 	return FALSE;		       /* so we'll do the next ones too */
     }