Attachment #194505 for bug #229222




	movq	%r8,PCPU(RSP0)
	movq	%r8,PCPU(CURPCB)
	/* Update the TSS_RSP0 pointer for the next interrupt */
	cmpq	$~0,PCPU(UCR3)
	je	1f
	movq	PCPU(PTI_RSP0),%rax
	movq	%rax,TSS_RSP0(%rdx)
	jmp	2f
1:	movq	%r8,TSS_RSP0(%rdx)
2:	movq	%r12,PCPU(CURTHREAD)		/* into next thread */

	/* Test if debug registers should be restored. */
	testl	$PCB_DBREGS,PCB_FLAGS(%r8)

	shrq	$8,%rcx
	movl	%ecx,8(%rax)
	movb	$0x89,5(%rax)	/* unset busy */
	movl	$TSSSEL,%eax
	je	1f
	movq	PCPU(PRVSPACE),%rax
	addq	$PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax
	movq	%rax,TSS_RSP0(%rdx)
1:	movl	$TSSSEL,%eax
	ltr	%ax
	jmp	done_tss


Lines 298-308 IDTVEC(page_pti) Link Here

(-)sys/amd64/amd64/exception.S (-8 / +16 lines)
298	jz Xpage	298	jz Xpage
299	swapgs	299	swapgs
300	pushq %rax	300	pushq %rax
301	pushq %rdx
302	movq %cr3,%rax	301	movq %cr3,%rax
303	movq %rax,PCPU(SAVED_UCR3)	302	movq %rax,PCPU(SAVED_UCR3)
		303	cmpq $~0,PCPU(UCR3)
		304	jne 1f
		305	popq %rax
		306	jmp 2f
		307	1: pushq %rdx
304	PTI_UUENTRY has_err=1	308	PTI_UUENTRY has_err=1
305	subq $TF_ERR,%rsp	309	2: subq $TF_ERR,%rsp
306	movq %rdi,TF_RDI(%rsp)	310	movq %rdi,TF_RDI(%rsp)
307	movq %rax,TF_RAX(%rsp)	311	movq %rax,TF_RAX(%rsp)
308	movq %rdx,TF_RDX(%rsp)	312	movq %rdx,TF_RDX(%rsp)
Lines 347-355 page_cr2: Link Here
347	*/	351	*/
348	.macro PROTF_ENTRY name,trapno	352	.macro PROTF_ENTRY name,trapno
349	\name\()_pti_doreti:	353	\name\()_pti_doreti:
		354	swapgs
		355	cmpq $~0,PCPU(UCR3)
		356	je 1f
350	pushq %rax	357	pushq %rax
351	pushq %rdx	358	pushq %rdx
352	swapgs
353	movq PCPU(KCR3),%rax	359	movq PCPU(KCR3),%rax
354	movq %rax,%cr3	360	movq %rax,%cr3
355	movq PCPU(RSP0),%rax	361	movq PCPU(RSP0),%rax
Lines 362-368 page_cr2: Link Here
362	movq %rax,%rsp	368	movq %rax,%rsp
363	popq %rdx	369	popq %rdx
364	popq %rax	370	popq %rax
365	swapgs	371	1: swapgs
366	jmp X\name	372	jmp X\name
367	IDTVEC(\name\()_pti)	373	IDTVEC(\name\()_pti)
368	cmpq $doreti_iret,PTI_RIP-2*8(%rsp)	374	cmpq $doreti_iret,PTI_RIP-2*8(%rsp)
Lines 438-443 prot_addrf: Link Here
438	IDTVEC(fast_syscall_pti)	444	IDTVEC(fast_syscall_pti)
439	swapgs	445	swapgs
440	movq %rax,PCPU(SCRATCH_RAX)	446	movq %rax,PCPU(SCRATCH_RAX)
		447	cmpq $~0,PCPU(UCR3)
		448	je fast_syscall_common
441	movq PCPU(KCR3),%rax	449	movq PCPU(KCR3),%rax
442	movq %rax,%cr3	450	movq %rax,%cr3
443	jmp fast_syscall_common	451	jmp fast_syscall_common
Lines 503-509 fast_syscall_common: Link Here
503	movq TF_RFLAGS(%rsp),%r11 /* original %rflags */	511	movq TF_RFLAGS(%rsp),%r11 /* original %rflags */
504	movq TF_RIP(%rsp),%rcx /* original %rip */	512	movq TF_RIP(%rsp),%rcx /* original %rip */
505	movq TF_RSP(%rsp),%rsp /* user stack pointer */	513	movq TF_RSP(%rsp),%rsp /* user stack pointer */
506	cmpb $0,pti	514	cmpq $~0,PCPU(UCR3)
507	je 2f	515	je 2f
508	movq PCPU(UCR3),%r9	516	movq PCPU(UCR3),%r9
509	movq %r9,%cr3	517	movq %r9,%cr3
Lines 1126-1136 ld_regs: Link Here
1126	jz 2f /* keep running with kernel GS.base */	1134	jz 2f /* keep running with kernel GS.base */
1127	cli	1135	cli
1128	call handle_ibrs_exit_rs	1136	call handle_ibrs_exit_rs
1129	cmpb $0,pti	1137	cmpq $~0,PCPU(UCR3)
1130	je 1f	1138	je 1f
1131	pushq %rdx	1139	pushq %rdx
1132	movq PCPU(PRVSPACE),%rdx	1140	movq PCPU(PTI_RSP0),%rdx
1133	addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx	1141	subq $PTI_SIZE,%rdx
1134	movq %rax,PTI_RAX(%rdx)	1142	movq %rax,PTI_RAX(%rdx)
1135	popq %rax	1143	popq %rax
1136	movq %rax,PTI_RDX(%rdx)	1144	movq %rax,PTI_RDX(%rdx)

Lines 229-234 ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3)); Link Here

(-)sys/amd64/amd64/genassym.c (+1 lines)
229	ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3));	229	ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3));
230	ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack));	230	ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack));
231	ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ);	231	ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ);
		232	ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0));
232	ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set));	233	ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set));
233		234
234	ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);	235	ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);

Lines 1817-1825 hammer_time(u_int64_t modulep, u_int64_t physfree) Link Here

(-)sys/amd64/amd64/machdep.c (-2 / +3 lines)
1817	rsp0 = (vm_offset_t)thread0.td_pcb;	1817	rsp0 = (vm_offset_t)thread0.td_pcb;
1818	/* Ensure the stack is aligned to 16 bytes */	1818	/* Ensure the stack is aligned to 16 bytes */
1819	rsp0 &= ~0xFul;	1819	rsp0 &= ~0xFul;
1820	common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) +	1820	common_tss[0].tss_rsp0 = rsp0;
1821	PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0;
1822	PCPU_SET(rsp0, rsp0);	1821	PCPU_SET(rsp0, rsp0);
		1822	PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) +
		1823	PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful);
1823	PCPU_SET(curpcb, thread0.td_pcb);	1824	PCPU_SET(curpcb, thread0.td_pcb);
1824		1825
1825	/* transfer to user mode */	1826	/* transfer to user mode */

Lines 255-260 init_secondary(void) Link Here

(-)sys/amd64/amd64/mp_machdep.c (-2 / +3 lines)
255	pc->pc_tssp = &common_tss[cpu];	255	pc->pc_tssp = &common_tss[cpu];
256	pc->pc_commontssp = &common_tss[cpu];	256	pc->pc_commontssp = &common_tss[cpu];
257	pc->pc_rsp0 = 0;	257	pc->pc_rsp0 = 0;
		258	pc->pc_pti_rsp0 = ((vm_offset_t)&pc->pc_pti_stack +
		259	PC_PTI_STACK_SZ * sizeof(uint64_t) & ~0xful);
258	pc->pc_tss = (struct system_segment_descriptor )&gdt[NGDT cpu +	260	pc->pc_tss = (struct system_segment_descriptor )&gdt[NGDT cpu +
259	GPROC0_SEL];	261	GPROC0_SEL];
260	pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL];	262	pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL];
Lines 264-271 init_secondary(void) Link Here
264	pc->pc_curpmap = kernel_pmap;	266	pc->pc_curpmap = kernel_pmap;
265	pc->pc_pcid_gen = 1;	267	pc->pc_pcid_gen = 1;
266	pc->pc_pcid_next = PMAP_PCID_KERN + 1;	268	pc->pc_pcid_next = PMAP_PCID_KERN + 1;
267	common_tss[cpu].tss_rsp0 = pti ? ((vm_offset_t)&pc->pc_pti_stack +	269	common_tss[cpu].tss_rsp0 = 0;
268	PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : 0;
269		270
270	/* Save the per-cpu pointer for use by the NMI handler. */	271	/* Save the per-cpu pointer for use by the NMI handler. */
271	np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1;	272	np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1;

Lines 2560-2567 pmap_pinit0(pmap_t pmap) Link Here

(-)sys/amd64/amd64/pmap.c (-3 / +13 lines)
2560	CPU_FOREACH(i) {	2560	CPU_FOREACH(i) {
2561	pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;	2561	pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
2562	pmap->pm_pcids[i].pm_gen = 0;	2562	pmap->pm_pcids[i].pm_gen = 0;
2563	if (!pti)	2563	if (!pti) {
2564	__pcpu[i].pc_kcr3 = PMAP_NO_CR3;	2564	__pcpu[i].pc_kcr3 = PMAP_NO_CR3;
		2565	__pcpu[i].pc_ucr3 = PMAP_NO_CR3;
		2566	}
2565	}	2567	}
2566	PCPU_SET(curpmap, kernel_pmap);	2568	PCPU_SET(curpmap, kernel_pmap);
2567	pmap_activate(curthread);	2569	pmap_activate(curthread);
Lines 2736-2742 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, Link Here
2736	* the kernel-mode page table active on return	2738	* the kernel-mode page table active on return
2737	* to user space.	2739	* to user space.
2738	*/	2740	*/
2739	*pml4 \|= pg_nx;	2741	if (pmap->pm_ucr3 != PMAP_NO_CR3)
		2742	*pml4 \|= pg_nx;
2740		2743
2741	pml4u = &pmap->pm_pml4u[pml4index];	2744	pml4u = &pmap->pm_pml4u[pml4index];
2742	*pml4u = VM_PAGE_TO_PHYS(m) \| PG_U \| PG_RW \| PG_V \|	2745	*pml4u = VM_PAGE_TO_PHYS(m) \| PG_U \| PG_RW \| PG_V \|
Lines 7337-7345 pmap_activate_sw(struct thread *td) Link Here
7337	{	7340	{
7338	pmap_t oldpmap, pmap;	7341	pmap_t oldpmap, pmap;
7339	struct invpcid_descr d;	7342	struct invpcid_descr d;
7340	uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3;	7343	uint64_t cached, cr3, kcr3, kern_pti_cached, rsp0, ucr3;
7341	register_t rflags;	7344	register_t rflags;
7342	u_int cpuid;	7345	u_int cpuid;
		7346	struct amd64tss *tssp;
7343		7347
7344	oldpmap = PCPU_GET(curpmap);	7348	oldpmap = PCPU_GET(curpmap);
7345	pmap = vmspace_pmap(td->td_proc->p_vmspace);	7349	pmap = vmspace_pmap(td->td_proc->p_vmspace);
Lines 7430-7435 pmap_activate_sw(struct thread *td) Link Here
7430	PCPU_SET(ucr3, pmap->pm_ucr3);	7434	PCPU_SET(ucr3, pmap->pm_ucr3);
7431	}	7435	}
7432	}	7436	}
		7437	if (pmap->pm_ucr3 != PMAP_NO_CR3) {
		7438	rsp0 = ((vm_offset_t)PCPU_PTR(pti_stack) +
		7439	PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful;
		7440	tssp = PCPU_GET(tssp);
		7441	tssp->tss_rsp0 = rsp0;
		7442	}
7433	#ifdef SMP	7443	#ifdef SMP
7434	CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);	7444	CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
7435	#else	7445	#else




			 */
			if (frame->tf_rip == (long)doreti_iret) {
				frame->tf_rip = (long)doreti_iret_fault;
				if ((PCPU_GET(curpmap)->pm_ucr3 !=
				    PMAP_NO_CR3) &&
				    (frame->tf_rsp == (uintptr_t)PCPU_GET(
				    pti_rsp0) - 5 * sizeof(register_t))) {
					frame->tf_rsp = PCPU_GET(rsp0) - 5 *
					    sizeof(register_t);
				}
				return;
			}
			if (frame->tf_rip == (long)ld_ds) {

Lines 194-202 Link Here

(-)sys/amd64/include/asmacros.h (+3 lines)
194		194
195	.macro PTI_UENTRY has_err	195	.macro PTI_UENTRY has_err
196	swapgs	196	swapgs
		197	cmpq $~0,PCPU(UCR3)
		198	je 1f
197	pushq %rax	199	pushq %rax
198	pushq %rdx	200	pushq %rdx
199	PTI_UUENTRY \has_err	201	PTI_UUENTRY \has_err
		202	1:
200	.endm	203	.endm
201		204
202	.macro PTI_ENTRY name, cont, has_err=0	205	.macro PTI_ENTRY name, cont, has_err=0

Lines 67-72 Link Here

(-)sys/amd64/include/pcpu.h (-1 / +2 lines)
67	u_int pc_cmci_mask; /* MCx banks for CMCI */ \	67	u_int pc_cmci_mask; /* MCx banks for CMCI */ \
68	uint64_t pc_dbreg[16]; /* ddb debugging regs */ \	68	uint64_t pc_dbreg[16]; /* ddb debugging regs */ \
69	uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \	69	uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \
		70	register_t pc_pti_rsp0; \
70	int pc_dbreg_cmd; /* ddb debugging reg cmd */ \	71	int pc_dbreg_cmd; /* ddb debugging reg cmd */ \
71	u_int pc_vcpu_id; /* Xen vCPU ID */ \	72	u_int pc_vcpu_id; /* Xen vCPU ID */ \
72	uint32_t pc_pcid_next; \	73	uint32_t pc_pcid_next; \
Lines 73-79 Link Here
73	uint32_t pc_pcid_gen; \	74	uint32_t pc_pcid_gen; \
74	uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \	75	uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
75	uint32_t pc_ibpb_set; \	76	uint32_t pc_ibpb_set; \
76	char __pad[96] /* be divisor of PAGE_SIZE \	77	char __pad[82] /* be divisor of PAGE_SIZE \
77	after cache alignment */	78	after cache alignment */
78		79
79	#define PC_DBREG_CMD_NONE 0	80	#define PC_DBREG_CMD_NONE 0

Return to bug 229222

Lines 463-473 trap(struct trapframe *frame) Link Here

(-)sys/amd64/amd64/trap.c (-3 / +5 lines)
463	*/	463	*/
464	if (frame->tf_rip == (long)doreti_iret) {	464	if (frame->tf_rip == (long)doreti_iret) {
465	frame->tf_rip = (long)doreti_iret_fault;	465	frame->tf_rip = (long)doreti_iret_fault;
466	if (pti && frame->tf_rsp == (uintptr_t)PCPU_PTR(	466	if ((PCPU_GET(curpmap)->pm_ucr3 !=
467	pti_stack) + (PC_PTI_STACK_SZ - 5) *	467	PMAP_NO_CR3) &&
468	sizeof(register_t))	468	(frame->tf_rsp == (uintptr_t)PCPU_GET(
		469	pti_rsp0) - 5 * sizeof(register_t))) {
469	frame->tf_rsp = PCPU_GET(rsp0) - 5 *	470	frame->tf_rsp = PCPU_GET(rsp0) - 5 *
470	sizeof(register_t);	471	sizeof(register_t);
		472	}
471	return;	473	return;
472	}	474	}
473	if (frame->tf_rip == (long)ld_ds) {	475	if (frame->tf_rip == (long)ld_ds) {

Lines 210-219 done_tss: Link Here

(-)sys/amd64/amd64/cpu_switch.S (-10 / +8 lines)
210	movq %r8,PCPU(RSP0)	210	movq %r8,PCPU(RSP0)
211	movq %r8,PCPU(CURPCB)	211	movq %r8,PCPU(CURPCB)
212	/* Update the TSS_RSP0 pointer for the next interrupt */	212	/* Update the TSS_RSP0 pointer for the next interrupt */
213	cmpb $0,pti(%rip)	213	cmpq $~0,PCPU(UCR3)
214	jne 1f	214	je 1f
215	movq %r8,TSS_RSP0(%rdx)	215	movq PCPU(PTI_RSP0),%rax
216	1: movq %r12,PCPU(CURTHREAD) /* into next thread */	216	movq %rax,TSS_RSP0(%rdx)
		217	jmp 2f
		218	1: movq %r8,TSS_RSP0(%rdx)
		219	2: movq %r12,PCPU(CURTHREAD) /* into next thread */
217		220
218	/* Test if debug registers should be restored. */	221	/* Test if debug registers should be restored. */
219	testl $PCB_DBREGS,PCB_FLAGS(%r8)	222	testl $PCB_DBREGS,PCB_FLAGS(%r8)
Lines 292-303 do_tss: movq %rdx,PCPU(TSSP) Link Here
292	shrq $8,%rcx	295	shrq $8,%rcx
293	movl %ecx,8(%rax)	296	movl %ecx,8(%rax)
294	movb $0x89,5(%rax) /* unset busy */	297	movb $0x89,5(%rax) /* unset busy */
295	cmpb $0,pti(%rip)	298	movl $TSSSEL,%eax
296	je 1f
297	movq PCPU(PRVSPACE),%rax
298	addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax
299	movq %rax,TSS_RSP0(%rdx)
300	1: movl $TSSSEL,%eax
301	ltr %ax	299	ltr %ax
302	jmp done_tss	300	jmp done_tss
303		301