diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index 89c5362784..431ec6e2fe 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -181,7 +181,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	}
 
 	// remove temporary podman files, images are now cached in ImageCacheDir
-	podman.removeCache(podman.TempDir)
+	rmAll(podman.PodmanBinary, podman.TempDir)
 
 	return []byte(path)
 }, func(data []byte) {
@@ -230,30 +230,21 @@ var _ = SynchronizedAfterSuite(func() {
 			GinkgoWriter.Printf("%s\t\t%f\n", result.name, result.length)
 		}
 
-		// previous runroot
-		tempdir, err := CreateTempDirInTempDir()
-		if err != nil {
-			os.Exit(1)
-		}
-		podmanTest := PodmanTestCreate(tempdir)
-		defer os.RemoveAll(tempdir)
-
-		if err := os.RemoveAll(podmanTest.Root); err != nil {
-			GinkgoWriter.Printf("%q\n", err)
-		}
-
-		// If running remote, we need to stop the associated podman system service
-		if podmanTest.RemoteTest {
-			podmanTest.StopRemoteService()
-		}
-		// for localized tests, this removes the image cache dir and for remote tests
-		// this is a no-op
-		podmanTest.removeCache(podmanTest.ImageCacheDir)
+		cwd, _ := os.Getwd()
+		rmAll(getPodmanBinary(cwd), ImageCacheDir)
 
 		// LockTmpDir can already be removed
 		os.RemoveAll(LockTmpDir)
 	})
 
+func getPodmanBinary(cwd string) string {
+	podmanBinary := filepath.Join(cwd, "../../bin/podman")
+	if os.Getenv("PODMAN_BINARY") != "" {
+		podmanBinary = os.Getenv("PODMAN_BINARY")
+	}
+	return podmanBinary
+}
+
 // PodmanTestCreate creates a PodmanTestIntegration instance for the tests
 func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 	var podmanRemoteBinary string
@@ -262,10 +253,7 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 	cwd, _ := os.Getwd()
 
 	root := filepath.Join(tempDir, "root")
-	podmanBinary := filepath.Join(cwd, "../../bin/podman")
-	if os.Getenv("PODMAN_BINARY") != "" {
-		podmanBinary = os.Getenv("PODMAN_BINARY")
-	}
+	podmanBinary := getPodmanBinary(cwd)
 
 	podmanRemoteBinary = filepath.Join(cwd, "../../bin/podman-remote")
 	if os.Getenv("PODMAN_REMOTE_BINARY") != "" {
@@ -654,7 +642,7 @@ func (p *PodmanTestIntegration) Cleanup() {
 
 	p.StopRemoteService()
 	// Nuke tempdir
-	p.removeCache(p.TempDir)
+	rmAll(p.PodmanBinary, p.TempDir)
 
 	// Clean up the registries configuration file ENV variable set in Create
 	resetRegistriesConfigEnv()
@@ -990,11 +978,13 @@ func populateCache(podman *PodmanTestIntegration) {
 	GinkgoWriter.Printf("-----------------------------\n")
 }
 
-func (p *PodmanTestIntegration) removeCache(path string) {
+// rmAll removes the direcory and its content,, when running rootless we use
+// podman unshare to prevent any subuid/gid problems
+func rmAll(podmanBin string, path string) {
 	// Remove cache dirs
 	if isRootless() {
 		// If rootless, os.RemoveAll() is failed due to permission denied
-		cmd := exec.Command(p.PodmanBinary, "unshare", "rm", "-rf", path)
+		cmd := exec.Command(podmanBin, "unshare", "rm", "-rf", path)
 		cmd.Stdout = GinkgoWriter
 		cmd.Stderr = GinkgoWriter
 		if err := cmd.Run(); err != nil {