diff --git a/mm/rmap.c b/mm/rmap.c
index 1235368f0628..795f9d5f8386 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1703,9 +1703,6 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
 	struct mmu_notifier_range range;
 	enum ttu_flags flags = (enum ttu_flags)(long)arg;
 
-	if (is_zone_device_page(page) && !is_device_private_page(page))
-		return true;
-
 	/*
 	 * When racing against e.g. zap_pte_range() on another cpu,
 	 * in between its ptep_get_and_clear_full() and page_remove_rmap(),
@@ -1944,6 +1941,9 @@ void try_to_migrate(struct page *page, enum ttu_flags flags)
 					TTU_SYNC)))
 		return;
 
+	if (is_zone_device_page(page) && !is_device_private_page(page))
+		return;
+
 	/*
 	 * During exec, a temporary VMA is setup and later moved.
 	 * The VMA is moved under the anon_vma lock but not the