diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index bad68991bb..a0f86afb50 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -1412,6 +1412,11 @@ func (c *Container) restartWithTimeout(ctx context.Context, timeout uint) (retEr
 		if err := c.stop(timeout); err != nil {
 			return err
 		}
+		if c.config.HealthCheckConfig != nil {
+			if err := c.removeTransientFiles(context.Background()); err != nil {
+				logrus.Error(err.Error())
+			}
+		}
 		// Old versions of conmon have a bug where they create the exit file before
 		// closing open file descriptors causing a race condition when restarting
 		// containers with open ports since we cannot bind the ports as they're not
diff --git a/test/system/220-healthcheck.bats b/test/system/220-healthcheck.bats
index 4c3ddbebfb..fc8994d185 100644
--- a/test/system/220-healthcheck.bats
+++ b/test/system/220-healthcheck.bats
@@ -125,6 +125,8 @@ Log[-1].Output   | \"Uh-oh on stdout!\\\nUh-oh on stderr!\"
 	    # kill and stop yield the container into a non-running state
             is "$output" ".* $policy" "container was stopped/killed"
             assert "$output" != "running $policy"
+            # also make sure that it's not stuck in the stopping state
+            assert "$output" != "stopping $policy"
         fi
 
         run_podman rm -f -t0 $ctr