diff --git a/xen/arch/x86/mm/p2m.c b/xen/arch/x86/mm/p2m.c
index 2433230ac7..759f3273d3 100644
--- a/xen/arch/x86/mm/p2m.c
+++ b/xen/arch/x86/mm/p2m.c
@@ -2015,11 +2015,17 @@ int xenmem_add_to_physmap_one(
     {
         gmfn = idx;
         mfn = get_gfn_unshare(d, gmfn, &p2mt);
-        /* If the page is still shared, exit early */
-        if ( p2m_is_shared(p2mt) )
+        /*
+         * The entry at the destination gfn will be created as type p2m_ram_rw.
+         * Only allow moving source gfns with read/write or logdirty RAM types
+         * to avoid unexpected p2m type changes as a result of the operation.
+         * Note that for logdirty source type we rely on p2m_add_page() marking
+         * the destination gfn as dirty.
+         */
+        if ( p2mt != p2m_ram_rw && p2mt != p2m_ram_logdirty )
         {
             put_gfn(d, gmfn);
-            return -ENOMEM;
+            return p2m_is_shared(p2mt) ? -ENOMEM : -EACCES;
         }
         page = get_page_from_mfn(mfn, d);
         if ( unlikely(!page) )