Attachment 238192 Details for Bug 266145 – Current experimental workaround

[patch] Current experimental workaround

1.patch (text/plain), 4.61 KB, created by Konstantin Belousov on 2022-11-20 15:04:44 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Konstantin Belousov

Created: 2022-11-20 15:04:44 UTC

Size: 4.61 KB

patch

obsolete

>commit 5d72240a8777b26d5e0a7d2d26bb919d05f60002
>Author: Konstantin Belousov <kib@FreeBSD.org>
>Date:   Tue Oct 11 02:08:55 2022 +0300
>
>    amd64: for small cores, use (big hammer) INVPCID_CTXGLOB instead of INVLPG
>
>diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c
>index 1a251208cfd6..36b55c4e7b2b 100644
>--- a/sys/amd64/amd64/initcpu.c
>+++ b/sys/amd64/amd64/initcpu.c
>@@ -255,6 +255,7 @@ initializecpu(void)
> {
> 	uint64_t msr;
> 	uint32_t cr4;
>+	u_int r[4];
> 
> 	cr4 = rcr4();
> 	if ((cpu_feature & CPUID_XMM) && (cpu_feature & CPUID_FXSR)) {
>@@ -317,6 +318,23 @@ initializecpu(void)
> 	if ((amd_feature & AMDID_RDTSCP) != 0 ||
> 	    (cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0)
> 		wrmsr(MSR_TSC_AUX, cpu_auxmsr());
>+
>+	if (cpu_high >= 0x1a) {
>+		cpuid_count(0x1a, 0, r);
>+		if ((r[0] & 0xff000000) == 0x20000000)
>+			PCPU_SET(small, 1);
>+	}
>+}
>+
>+void
>+invlpgXX(vm_offset_t va)
>+{
>+	if (PCPU_GET(small)) {
>+		struct invpcid_descr d = { 0 };
>+		invpcid(&d, INVPCID_CTXGLOB);
>+	} else {
>+		invlpg(va);
>+	}
> }
> 
> void
>diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
>index 2ed3a57866a2..e1e6cc90ba74 100644
>--- a/sys/amd64/amd64/mp_machdep.c
>+++ b/sys/amd64/amd64/mp_machdep.c
>@@ -864,7 +864,10 @@ invlpg_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
> 	(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
> #endif /* COUNT_IPIS */
> 
>-	invlpg(smp_tlb_addr1);
>+	if (smp_tlb_pmap == kernel_pmap)
>+		invlpgXX(smp_tlb_addr1);
>+	else
>+		invlpg(smp_tlb_addr1);
> 	if (smp_tlb_pmap == PCPU_GET(curpmap) &&
> 	    smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
> 	    PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
>@@ -935,7 +938,10 @@ invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
> 
> 	addr = smp_tlb_addr1;
> 	do {
>-		invlpg(addr);
>+		if (smp_tlb_pmap == kernel_pmap)
>+			invlpgXX(addr);
>+		else
>+			invlpg(addr);
> 		addr += PAGE_SIZE;
> 	} while (addr < smp_tlb_addr2);
> 	if (smp_tlb_pmap == PCPU_GET(curpmap) &&
>diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
>index 82f80d207130..1557c098aa12 100644
>--- a/sys/amd64/amd64/pmap.c
>+++ b/sys/amd64/amd64/pmap.c
>@@ -2797,7 +2797,7 @@ pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde)
> 
> 	if ((newpde & PG_PS) == 0)
> 		/* Demotion: flush a specific 2MB page mapping. */
>-		invlpg(va);
>+		invlpgXX(va);
> 	else if ((newpde & PG_G) == 0)
> 		/*
> 		 * Promotion: flush every 4KB page mapping from the TLB
>@@ -3136,7 +3136,7 @@ pmap_invalidate_page_curcpu_cb(pmap_t pmap, vm_offset_t va,
>     vm_offset_t addr2 __unused)
> {
> 	if (pmap == kernel_pmap) {
>-		invlpg(va);
>+		invlpgXX(va);
> 	} else if (pmap == PCPU_GET(curpmap)) {
> 		invlpg(va);
> 		pmap_invalidate_page_cb(pmap, va);
>@@ -3228,7 +3228,7 @@ pmap_invalidate_range_curcpu_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
> 
> 	if (pmap == kernel_pmap) {
> 		for (addr = sva; addr < eva; addr += PAGE_SIZE)
>-			invlpg(addr);
>+			invlpgXX(addr);
> 	} else if (pmap == PCPU_GET(curpmap)) {
> 		for (addr = sva; addr < eva; addr += PAGE_SIZE)
> 			invlpg(addr);
>@@ -7651,7 +7651,7 @@ pmap_kenter_temporary(vm_paddr_t pa, int i)
> 
> 	va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
> 	pmap_kenter(va, pa);
>-	invlpg(va);
>+	invlpgXX(va);
> 	return ((void *)crashdumpmap);
> }
> 
>@@ -10354,7 +10354,7 @@ pmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count,
> 				    page[i]->md.pat_mode, 0);
> 				pte_store(pte, paddr | X86_PG_RW | X86_PG_V |
> 				    cache_bits);
>-				invlpg(vaddr[i]);
>+				invlpgXX(vaddr[i]);
> 			}
> 		}
> 	}
>@@ -10403,7 +10403,7 @@ pmap_quick_remove_page(vm_offset_t addr)
> 	if (addr != qframe)
> 		return;
> 	pte_store(vtopte(qframe), 0);
>-	invlpg(qframe);
>+	invlpgXX(qframe);
> 	mtx_unlock_spin(&qframe_mtx);
> }
> 
>diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h
>index f014c66c0d06..3d961b2266cf 100644
>--- a/sys/amd64/include/md_var.h
>+++ b/sys/amd64/include/md_var.h
>@@ -99,6 +99,7 @@ void	get_fpcontext(struct thread *td, struct __mcontext *mcp,
> int	set_fpcontext(struct thread *td, struct __mcontext *mcp,
> 	    char *xfpustate, size_t xfpustate_len);
> 
>+void invlpgXX(vm_offset_t va);
> #endif /* !_MACHINE_MD_VAR_H_ */
> 
> #endif /* __i386__ */
>diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
>index 8dcf51cc71c6..5842e058458c 100644
>--- a/sys/amd64/include/pcpu.h
>+++ b/sys/amd64/include/pcpu.h
>@@ -99,7 +99,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
> 	uint32_t pc_smp_tlb_gen;					\
> 	u_int	pc_smp_tlb_op;						\
> 	uint64_t pc_ucr3_load_mask;					\
>-	char	__pad[2916]		/* pad to UMA_PCPU_ALLOC_SIZE */
>+	u_int	pc_small;						\
>+	char	__pad[2912]		/* pad to UMA_PCPU_ALLOC_SIZE */
> 
> #define	PC_DBREG_CMD_NONE	0
> #define	PC_DBREG_CMD_LOAD	1

Actions: View | Diff

Attachments on bug 266145: 236272 | 238192