diff --git a/mm/memory.c b/mm/memory.c
index 62fba9f3fd67..739f0983bb8c 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3601,6 +3601,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	vm_fault_t ret;
 	void *shadow = NULL;
 
+	if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
+		pte_unmap(vmf->pte);
+		return VM_FAULT_RETRY;
+	}
+
 	ret = pte_unmap_same(vmf);
 	if (ret) {
 		/*