Attachment #210691 for bug #242137




	pmap_t pmap;
	pv_entry_t next_pv, pv;
	pd_entry_t oldpde, *pde;
	pt_entry_t *pte, PG_M, PG_RW;
	struct rwlock *lock;
	vm_offset_t va;
	int md_gen, pvh_gen;

			}
		}
		PG_M = pmap_modified_bit(pmap);
		PG_V = pmap_valid_bit(pmap);
		PG_RW = pmap_rw_bit(pmap);
		va = pv->pv_va;
		pde = pmap_pde(pmap, va);
		oldpde = *pde;
		/* If oldpde has PG_RW set, then it also has PG_M set. */
		if ((oldpde & PG_RW) != 0 &&
		    pmap_demote_pde_locked(pmap, pde, va, &lock) &&
		    (oldpde & PG_W) == 0) {
			/*
			 * Write protect the mapping to a single page so that
			 * a subsequent write access may repromote.
			 */
			va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
			pte = pmap_pde_to_pte(pde, va);
			atomic_clear_long(pte, PG_M | PG_RW);
			vm_page_dirty(m);
			pmap_invalidate_page(pmap, va);
						while (!atomic_cmpset_long(pte,
						    oldpte,
						    oldpte & ~(PG_M | PG_RW)))
							oldpte = *pte;
						vm_page_dirty(m);
						pmap_invalidate_page(pmap, va);
					}
				}
			}
		}
		PMAP_UNLOCK(pmap);
	}




		va = pv->pv_va;
		l2 = pmap_l2(pmap, va);
		oldl2 = pmap_load(l2);
		/* If oldl2 has ATTR_SW_DBM set, then it is also dirty. */
		if ((oldl2 & ATTR_SW_DBM) != 0 &&
		    pmap_demote_l2_locked(pmap, l2, va, &lock) &&
		    (oldl2 & ATTR_SW_WIRED) == 0) {
			/*
			 * Write protect the mapping to a single page so that
			 * a subsequent write access may repromote.
			 */
			va += VM_PAGE_TO_PHYS(m) - (oldl2 & ~ATTR_MASK);
			l3 = pmap_l2_to_l3(l2, va);
			oldl3 = pmap_load(l3);
			while (!atomic_fcmpset_long(l3, &oldl3,
			    (oldl3 & ~ATTR_SW_DBM) | ATTR_AP(ATTR_AP_RO)))
				cpu_spinwait();
			vm_page_dirty(m);
			pmap_invalidate_page(pmap, va);
							cpu_spinwait();
						vm_page_dirty(m);
						pmap_invalidate_page(pmap, va);
					}
				}
			}
		}
		PMAP_UNLOCK(pmap);
	}




	pv_entry_t next_pv, pv;
	pmap_t pmap;
	pd_entry_t oldpde, *pde;
	pt_entry_t *pte;
	vm_offset_t va;

	KASSERT((m->oflags & VPO_UNMANAGED) == 0,

		PMAP_LOCK(pmap);
		pde = pmap_pde(pmap, va);
		oldpde = *pde;
		/* If oldpde has PG_RW set, then it also has PG_M set. */
		if ((oldpde & PG_RW) != 0 &&
		    pmap_demote_pde(pmap, pde, va) &&
		    (oldpde & PG_W) == 0) {
			/*
			 * Write protect the mapping to a single page so that
			 * a subsequent write access may repromote.
			 */
			va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
			pte = pmap_pte_quick(pmap, va);
			/*
			 * Regardless of whether a pte is 32 or 64 bits
			 * in size, PG_RW and PG_M are among the least
			 * significant 32 bits.
			 */
			atomic_clear_int((u_int *)pte, PG_M | PG_RW);
			vm_page_dirty(m);
			pmap_invalidate_page(pmap, va);
						while (!atomic_cmpset_int((u_int *)pte,
						    oldpte,
						    oldpte & ~(PG_M | PG_RW)))
							oldpte = *pte;
						vm_page_dirty(m);
						pmap_invalidate_page(pmap, va);
					}
				}
			}
		}
		PMAP_UNLOCK(pmap);
	}




	pmap_t pmap;
	pv_entry_t next_pv, pv;
	pd_entry_t *l2, oldl2;
	pt_entry_t *l3;
	vm_offset_t va;
	int md_gen, pvh_gen;


		va = pv->pv_va;
		l2 = pmap_l2(pmap, va);
		oldl2 = pmap_load(l2);
		/* If oldl2 has PTE_W set, then it also has PTE_D set. */
		if ((oldl2 & PTE_W) != 0 &&
		    pmap_demote_l2_locked(pmap, l2, va, &lock) &&
		    (oldl2 & PTE_SW_WIRED) == 0) {
			/*
			 * Write protect the mapping to a single page so that
			 * a subsequent write access may repromote.
			 */
			va += VM_PAGE_TO_PHYS(m) - PTE_TO_PHYS(oldl2);
			l3 = pmap_l2_to_l3(l2, va);
			pmap_clear_bits(l3, PTE_D | PTE_W);
			vm_page_dirty(m);
			pmap_invalidate_page(pmap, va);
						while (!atomic_fcmpset_long(l3,
						    &oldl3, oldl3 & ~(PTE_D |
						    PTE_W)))
							cpu_spinwait();
						vm_page_dirty(m);
						pmap_invalidate_page(pmap, va);
					}
				}
			}
		}
		PMAP_UNLOCK(pmap);
	}

Return to bug 242137

Lines 7599-7605 Link Here

(-)sys/amd64/amd64/pmap.c (-24 / +14 lines)
7599	pmap_t pmap;	7599	pmap_t pmap;
7600	pv_entry_t next_pv, pv;	7600	pv_entry_t next_pv, pv;
7601	pd_entry_t oldpde, *pde;	7601	pd_entry_t oldpde, *pde;
7602	pt_entry_t oldpte, *pte, PG_M, PG_RW, PG_V;	7602	pt_entry_t *pte, PG_M, PG_RW;
7603	struct rwlock *lock;	7603	struct rwlock *lock;
7604	vm_offset_t va;	7604	vm_offset_t va;
7605	int md_gen, pvh_gen;	7605	int md_gen, pvh_gen;
Lines 7635-7667 Link Here
7635	}	7635	}
7636	}	7636	}
7637	PG_M = pmap_modified_bit(pmap);	7637	PG_M = pmap_modified_bit(pmap);
7638	PG_V = pmap_valid_bit(pmap);
7639	PG_RW = pmap_rw_bit(pmap);	7638	PG_RW = pmap_rw_bit(pmap);
7640	va = pv->pv_va;	7639	va = pv->pv_va;
7641	pde = pmap_pde(pmap, va);	7640	pde = pmap_pde(pmap, va);
7642	oldpde = *pde;	7641	oldpde = *pde;
7643	if ((oldpde & PG_RW) != 0) {	7642	/* If oldpde has PG_RW set, then it also has PG_M set. */
7644	if (pmap_demote_pde_locked(pmap, pde, va, &lock)) {	7643	if ((oldpde & PG_RW) != 0 &&
7645	if ((oldpde & PG_W) == 0) {	7644	pmap_demote_pde_locked(pmap, pde, va, &lock) &&
7646	/*	7645	(oldpde & PG_W) == 0) {
7647	* Write protect the mapping to a	7646	/*
7648	* single page so that a subsequent	7647	* Write protect the mapping to a single page so that
7649	* write access may repromote.	7648	* a subsequent write access may repromote.
7650	*/	7649	*/
7651	va += VM_PAGE_TO_PHYS(m) - (oldpde &	7650	va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
7652	PG_PS_FRAME);	7651	pte = pmap_pde_to_pte(pde, va);
7653	pte = pmap_pde_to_pte(pde, va);	7652	atomic_clear_long(pte, PG_M \| PG_RW);
7654	oldpte = *pte;	7653	vm_page_dirty(m);
7655	if ((oldpte & PG_V) != 0) {	7654	pmap_invalidate_page(pmap, va);
7656	while (!atomic_cmpset_long(pte,
7657	oldpte,
7658	oldpte & ~(PG_M \| PG_RW)))
7659	oldpte = *pte;
7660	vm_page_dirty(m);
7661	pmap_invalidate_page(pmap, va);
7662	}
7663	}
7664	}
7665	}	7655	}
7666	PMAP_UNLOCK(pmap);	7656	PMAP_UNLOCK(pmap);
7667	}	7657	}

Lines 5008-5035 Link Here

(-)sys/arm64/arm64/pmap.c (-22 / +16 lines)
5008	va = pv->pv_va;	5008	va = pv->pv_va;
5009	l2 = pmap_l2(pmap, va);	5009	l2 = pmap_l2(pmap, va);
5010	oldl2 = pmap_load(l2);	5010	oldl2 = pmap_load(l2);
5011	if ((oldl2 & ATTR_SW_DBM) != 0) {	5011	/* If oldl2 has ATTR_SW_DBM set, then it is also dirty. */
5012	if (pmap_demote_l2_locked(pmap, l2, va, &lock)) {	5012	if ((oldl2 & ATTR_SW_DBM) != 0 &&
5013	if ((oldl2 & ATTR_SW_WIRED) == 0) {	5013	pmap_demote_l2_locked(pmap, l2, va, &lock) &&
5014	/*	5014	(oldl2 & ATTR_SW_WIRED) == 0) {
5015	* Write protect the mapping to a	5015	/*
5016	* single page so that a subsequent	5016	* Write protect the mapping to a single page so that
5017	* write access may repromote.	5017	* a subsequent write access may repromote.
5018	*/	5018	*/
5019	va += VM_PAGE_TO_PHYS(m) -	5019	va += VM_PAGE_TO_PHYS(m) - (oldl2 & ~ATTR_MASK);
5020	(oldl2 & ~ATTR_MASK);	5020	l3 = pmap_l2_to_l3(l2, va);
5021	l3 = pmap_l2_to_l3(l2, va);	5021	oldl3 = pmap_load(l3);
5022	oldl3 = pmap_load(l3);	5022	while (!atomic_fcmpset_long(l3, &oldl3,
5023	if (pmap_l3_valid(oldl3)) {	5023	(oldl3 & ~ATTR_SW_DBM) \| ATTR_AP(ATTR_AP_RO)))
5024	while (!atomic_fcmpset_long(l3,	5024	cpu_spinwait();
5025	&oldl3, (oldl3 & ~ATTR_SW_DBM) \|	5025	vm_page_dirty(m);
5026	ATTR_AP(ATTR_AP_RO)))	5026	pmap_invalidate_page(pmap, va);
5027	cpu_spinwait();
5028	vm_page_dirty(m);
5029	pmap_invalidate_page(pmap, va);
5030	}
5031	}
5032	}
5033	}	5027	}
5034	PMAP_UNLOCK(pmap);	5028	PMAP_UNLOCK(pmap);
5035	}	5029	}

Lines 5366-5372 Link Here

(-)sys/i386/i386/pmap.c (-28 / +19 lines)
5366	pv_entry_t next_pv, pv;	5366	pv_entry_t next_pv, pv;
5367	pmap_t pmap;	5367	pmap_t pmap;
5368	pd_entry_t oldpde, *pde;	5368	pd_entry_t oldpde, *pde;
5369	pt_entry_t oldpte, *pte;	5369	pt_entry_t *pte;
5370	vm_offset_t va;	5370	vm_offset_t va;
5371		5371
5372	KASSERT((m->oflags & VPO_UNMANAGED) == 0,	5372	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
Lines 5393-5425 Link Here
5393	PMAP_LOCK(pmap);	5393	PMAP_LOCK(pmap);
5394	pde = pmap_pde(pmap, va);	5394	pde = pmap_pde(pmap, va);
5395	oldpde = *pde;	5395	oldpde = *pde;
5396	if ((oldpde & PG_RW) != 0) {	5396	/* If oldpde has PG_RW set, then it also has PG_M set. */
5397	if (pmap_demote_pde(pmap, pde, va)) {	5397	if ((oldpde & PG_RW) != 0 &&
5398	if ((oldpde & PG_W) == 0) {	5398	pmap_demote_pde(pmap, pde, va) &&
5399	/*	5399	(oldpde & PG_W) == 0) {
5400	* Write protect the mapping to a	5400	/*
5401	* single page so that a subsequent	5401	* Write protect the mapping to a single page so that
5402	* write access may repromote.	5402	* a subsequent write access may repromote.
5403	*/	5403	*/
5404	va += VM_PAGE_TO_PHYS(m) - (oldpde &	5404	va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
5405	PG_PS_FRAME);	5405	pte = pmap_pte_quick(pmap, va);
5406	pte = pmap_pte_quick(pmap, va);	5406	/*
5407	oldpte = *pte;	5407	* Regardless of whether a pte is 32 or 64 bits
5408	if ((oldpte & PG_V) != 0) {	5408	* in size, PG_RW and PG_M are among the least
5409	/*	5409	* significant 32 bits.
5410	* Regardless of whether a pte is 32 or 64 bits	5410	*/
5411	* in size, PG_RW and PG_M are among the least	5411	atomic_clear_int((u_int *)pte, PG_M \| PG_RW);
5412	* significant 32 bits.	5412	vm_page_dirty(m);
5413	*/	5413	pmap_invalidate_page(pmap, va);
5414	while (!atomic_cmpset_int((u_int *)pte,
5415	oldpte,
5416	oldpte & ~(PG_M \| PG_RW)))
5417	oldpte = *pte;
5418	vm_page_dirty(m);
5419	pmap_invalidate_page(pmap, va);
5420	}
5421	}
5422	}
5423	}	5414	}
5424	PMAP_UNLOCK(pmap);	5415	PMAP_UNLOCK(pmap);
5425	}	5416	}

Lines 4104-4110 Link Here

(-)sys/riscv/riscv/pmap.c (-23 / +14 lines)
4104	pmap_t pmap;	4104	pmap_t pmap;
4105	pv_entry_t next_pv, pv;	4105	pv_entry_t next_pv, pv;
4106	pd_entry_t *l2, oldl2;	4106	pd_entry_t *l2, oldl2;
4107	pt_entry_t *l3, oldl3;	4107	pt_entry_t *l3;
4108	vm_offset_t va;	4108	vm_offset_t va;
4109	int md_gen, pvh_gen;	4109	int md_gen, pvh_gen;
4110		4110
Lines 4142-4169 Link Here
4142	va = pv->pv_va;	4142	va = pv->pv_va;
4143	l2 = pmap_l2(pmap, va);	4143	l2 = pmap_l2(pmap, va);
4144	oldl2 = pmap_load(l2);	4144	oldl2 = pmap_load(l2);
4145	if ((oldl2 & PTE_W) != 0) {	4145	/* If oldl2 has PTE_W set, then it also has PTE_D set. */
4146	if (pmap_demote_l2_locked(pmap, l2, va, &lock)) {	4146	if ((oldl2 & PTE_W) != 0 &&
4147	if ((oldl2 & PTE_SW_WIRED) == 0) {	4147	pmap_demote_l2_locked(pmap, l2, va, &lock) &&
4148	/*	4148	(oldl2 & PTE_SW_WIRED) == 0) {
4149	* Write protect the mapping to a	4149	/*
4150	* single page so that a subsequent	4150	* Write protect the mapping to a single page so that
4151	* write access may repromote.	4151	* a subsequent write access may repromote.
4152	*/	4152	*/
4153	va += VM_PAGE_TO_PHYS(m) -	4153	va += VM_PAGE_TO_PHYS(m) - PTE_TO_PHYS(oldl2);
4154	PTE_TO_PHYS(oldl2);	4154	l3 = pmap_l2_to_l3(l2, va);
4155	l3 = pmap_l2_to_l3(l2, va);	4155	pmap_clear_bits(l3, PTE_D \| PTE_W);
4156	oldl3 = pmap_load(l3);	4156	vm_page_dirty(m);
4157	if ((oldl3 & PTE_V) != 0) {	4157	pmap_invalidate_page(pmap, va);
4158	while (!atomic_fcmpset_long(l3,
4159	&oldl3, oldl3 & ~(PTE_D \|
4160	PTE_W)))
4161	cpu_spinwait();
4162	vm_page_dirty(m);
4163	pmap_invalidate_page(pmap, va);
4164	}
4165	}
4166	}
4167	}	4158	}
4168	PMAP_UNLOCK(pmap);	4159	PMAP_UNLOCK(pmap);
4169	}	4160	}