From: Matthew Dillon <dillon@apollo.backplane.com>
Date: Mon, 12 Dec 2011 17:37:20 +0000 (-0800)
Subject: kernel - Misc fixes and debugging
X-Git-Tag: v3.0.0~418
X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/3338cc67bf8b657fca8ae9544aa3f89bb32cffb8

kernel - Misc fixes and debugging

* Add required CLDs in the exception paths.  The interrupt paths already
  CLD in PUSH_FRAME.

* Note that the fast_syscall (SYSENTER) path has an implied CLD due to the
  hardware mask applied to rflags.

* Add the IOPL bits to the set of bits set to 0 during a fast_syscall.

* When creating a dummy interrupt frame we don't have to push the
  actual %cs.  Just push $0 so the frame isn't misinterpreted as coming
  from userland.

* Additional debug verbosity for freeze_on_seg_fault.

* Reserve two void * fields for LWP debugging (for a later commit)
---

diff --git a/sys/platform/pc32/i386/exception.s b/sys/platform/pc32/i386/exception.s
index 987687fef0..b0039c84eb 100644
--- a/sys/platform/pc32/i386/exception.s
+++ b/sys/platform/pc32/i386/exception.s
@@ -769,7 +769,6 @@ alltraps:
 	pushl	%es
 	pushl	%fs
 	pushl	%gs
-	cld
 	.globl	alltraps_with_regs_pushed
 alltraps_with_regs_pushed:
 	mov	$KDSEL,%ax
@@ -783,6 +782,7 @@ calltrap:
 	FAKE_MCOUNT(btrap)		/* init "from" _btrap -> calltrap */
 	incl	PCPU(cnt)+V_TRAP
 	/* warning, trap frame dummy arg, no extra reg pushes */
+	cld
 	pushl	%esp			/* pass frame by reference */
 	call	trap
 	addl	$4,%esp
diff --git a/sys/platform/pc64/apic/apic_vector.s b/sys/platform/pc64/apic/apic_vector.s
index f9de780739..b338bce6a4 100644
--- a/sys/platform/pc64/apic/apic_vector.s
+++ b/sys/platform/pc64/apic/apic_vector.s
@@ -52,7 +52,8 @@
  * segment register being changed (e.g. by procfs), which is why syscalls
  * have to use doreti.
  */
-#define APIC_POP_FRAME POP_FRAME
+#define APIC_POP_FRAME							\
+	POP_FRAME ;							\
 
 #define IOAPICADDR(irq_num) \
 	CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
@@ -161,18 +162,24 @@ IDTVEC(ioapic_intr##irq_num) ;						\
 
 /*
  * Handle "spurious INTerrupts".
- * Notes:
- *  This is different than the "spurious INTerrupt" generated by an
- *   8259 PIC for missing INTs.  See the APIC documentation for details.
- *  This routine should NOT do an 'EOI' cycle.
+ *
+ * NOTE: This is different than the "spurious INTerrupt" generated by an
+ *	 8259 PIC for missing INTs.  See the APIC documentation for details.
+ *	 This routine should NOT do an 'EOI' cycle.
+ *
+ * NOTE: Even though we don't do anything here we must still swapgs if
+ *	 coming from a user frame in case the iretq faults... just use
+ *	 the nominal APIC_PUSH_FRAME sequence to get it done.
  */
 	.text
 	SUPERALIGN_TEXT
 	.globl Xspuriousint
 Xspuriousint:
-
+	APIC_PUSH_FRAME
 	/* No EOI cycle used here */
-
+	FAKE_MCOUNT(TF_RIP(%rsp))
+	MEXITCOUNT
+	APIC_POP_FRAME
 	jmp	doreti_iret
 
 #ifdef SMP
diff --git a/sys/platform/pc64/x86_64/exception.S b/sys/platform/pc64/x86_64/exception.S
index 81106acc5c..52bb3a6a0d 100644
--- a/sys/platform/pc64/x86_64/exception.S
+++ b/sys/platform/pc64/x86_64/exception.S
@@ -183,6 +183,7 @@ alltraps_pushregs_no_rdi:
 	.globl	calltrap
 	.type	calltrap,@function
 calltrap:
+	cld
 	movq	%rsp, %rdi
 	call	trap
 	MEXITCOUNT
@@ -228,6 +229,7 @@ IDTVEC(dblfault)
 	jz	1f
 	swapgs
 1:	movq	%rsp, %rdi
+	cld
 	call	dblfault_handler
 2:	hlt
 	jmp	2b
@@ -409,6 +411,7 @@ nmi_needswapgs:
 /* Note: this label is also used by ddb and gdb: */
 nmi_calltrap:
 	FAKE_MCOUNT(TF_RIP(%rsp))
+	cld
 	movq	%rsp, %rdi
 	call	trap
 	MEXITCOUNT
diff --git a/sys/platform/pc64/x86_64/ipl.s b/sys/platform/pc64/x86_64/ipl.s
index ff8f4282df..a16a35cd9e 100644
--- a/sys/platform/pc64/x86_64/ipl.s
+++ b/sys/platform/pc64/x86_64/ipl.s
@@ -474,7 +474,7 @@ splz_timer:
 	 */
 #define PUSH_DUMMY							\
 	pushfq ;			/* phys int frame / flags */	\
-	movl	%cs,%eax ;						\
+	xorq	%rax,%rax ;		/* something not SEL_UPL */	\
 	pushq	%rax ;			/* phys int frame / cs */	\
 	pushq	3*8(%rsp) ;		/* original caller eip */	\
 	subq	$TF_RIP,%rsp ;		/* trap frame */		\
diff --git a/sys/platform/pc64/x86_64/machdep.c b/sys/platform/pc64/x86_64/machdep.c
index 889a39883d..bbf3068c85 100644
--- a/sys/platform/pc64/x86_64/machdep.c
+++ b/sys/platform/pc64/x86_64/machdep.c
@@ -1916,7 +1916,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
 	      ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
 	wrmsr(MSR_STAR, msr);
-	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
+	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_IOPL);
 
 	getmemsize(kmdp, physfree);
 	init_param2(physmem);
diff --git a/sys/platform/pc64/x86_64/mp_machdep.c b/sys/platform/pc64/x86_64/mp_machdep.c
index 8a9386165d..e36fa998ee 100644
--- a/sys/platform/pc64/x86_64/mp_machdep.c
+++ b/sys/platform/pc64/x86_64/mp_machdep.c
@@ -290,7 +290,7 @@ init_secondary(void)
 	msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
 	      ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
 	wrmsr(MSR_STAR, msr);
-	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
+	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_IOPL);
 
 	pmap_set_opt();		/* PSE/4MB pages, etc */
 #if JGXXX
diff --git a/sys/platform/pc64/x86_64/swtch.s b/sys/platform/pc64/x86_64/swtch.s
index 14b543c43e..7c0c244103 100644
--- a/sys/platform/pc64/x86_64/swtch.s
+++ b/sys/platform/pc64/x86_64/swtch.s
@@ -163,6 +163,7 @@ ENTRY(cpu_heavy_switch)
 	 * PCB_RSP.  We push the flags for later restore by cpu_heavy_restore.
 	 */
 	pushfq
+	cli
 	movq	$cpu_heavy_restore, %rax
 	pushq	%rax
 	movq	%rsp,TD_SP(%rbx)
@@ -267,6 +268,7 @@ ENTRY(cpu_exit_switch)
 	 * thread but %rsp still points to the old thread's stack, but
 	 * we are protected by a critical section so it is ok.
 	 */
+	cli
 	movq	%rdi,%rax
 	movq	%rax,PCPU(curthread)
 	movq	TD_SP(%rax),%rsp
@@ -478,6 +480,8 @@ ENTRY(cpu_heavy_restore)
 	movq	PCB_R15(%rdx), %r15
 	movq	PCB_RIP(%rdx), %rax
 	movq	%rax, (%rsp)
+	movw	$KDSEL,%ax
+	movw	%ax,%es
 
 #if JG
 	/*
@@ -704,6 +708,7 @@ ENTRY(cpu_lwkt_switch)
 	pushq	%r14
 	pushq	%r15
 	pushfq
+	cli
 
 #if 1
 	/*
diff --git a/sys/platform/pc64/x86_64/trap.c b/sys/platform/pc64/x86_64/trap.c
index 3a936d10bb..70c48c1692 100644
--- a/sys/platform/pc64/x86_64/trap.c
+++ b/sys/platform/pc64/x86_64/trap.c
@@ -937,11 +937,11 @@ nogo:
 	if (td->td_lwp->lwp_vkernel == NULL) {
 		if (bootverbose || freeze_on_seg_fault || ddb_on_seg_fault) {
 			kprintf("seg-fault ft=%04x ff=%04x addr=%p rip=%p "
-			    "pid=%d p_comm=%s\n",
+			    "pid=%d cpu=%d p_comm=%s\n",
 			    ftype, fault_flags,
 			    (void *)frame->tf_addr,
 			    (void *)frame->tf_rip,
-			    p->p_pid, p->p_comm);
+			    p->p_pid, mycpu->gd_cpuid, p->p_comm);
 		}
 #ifdef DDB
 		while (freeze_on_seg_fault) {
@@ -1276,6 +1276,9 @@ out:
 		 * (which was holding the value of %rcx) is restored for
 		 * the next iteration.
 		 */
+		if (frame->tf_err != 0 && frame->tf_err != 2)
+			kprintf("lp %s:%d frame->tf_err is weird %ld\n",
+				td->td_comm, lp->lwp_proc->p_pid, frame->tf_err);
 		frame->tf_rip -= frame->tf_err;
 		frame->tf_r10 = frame->tf_rcx;
 		break;
diff --git a/sys/platform/pc64/x86_64/vm_machdep.c b/sys/platform/pc64/x86_64/vm_machdep.c
index 1e46db3db9..0a3797dcfb 100644
--- a/sys/platform/pc64/x86_64/vm_machdep.c
+++ b/sys/platform/pc64/x86_64/vm_machdep.c
@@ -201,7 +201,8 @@ cpu_prepare_lwp(struct lwp *lp, struct lwp_params *params)
 	regs->tf_rip = (long)params->func;
 	regs->tf_rsp = (long)params->stack;
 	/* Set up argument for function call */
-	regs->tf_rdi = (long)params->arg; /* JG Can this be in userspace addresses? */
+	regs->tf_rdi = (long)params->arg;
+
 	/*
 	 * Set up fake return address.  As the lwp function may never return,
 	 * we simply copy out a NULL pointer and force the lwp to receive
@@ -410,3 +411,68 @@ kvm_access_check(vm_offset_t saddr, vm_offset_t eaddr, int prot)
 	return 0;
 }
 
+#if 0
+
+void _test_frame_enter(struct trapframe *frame);
+void _test_frame_exit(struct trapframe *frame);
+
+void
+_test_frame_enter(struct trapframe *frame)
+{
+	thread_t td = curthread;
+
+	if (ISPL(frame->tf_cs) == SEL_UPL) {
+		KKASSERT(td->td_lwp);
+                KASSERT(td->td_lwp->lwp_md.md_regs == frame,
+                        ("_test_frame_exit: Frame mismatch %p %p",
+			td->td_lwp->lwp_md.md_regs, frame));
+	    td->td_lwp->lwp_saveusp = (void *)frame->tf_rsp;
+	    td->td_lwp->lwp_saveupc = (void *)frame->tf_rip;
+	}
+	if ((char *)frame < td->td_kstack ||
+	    (char *)frame > td->td_kstack + td->td_kstack_size) {
+		panic("_test_frame_exit: frame not on kstack %p kstack=%p\n",
+			frame, td->td_kstack);
+	}
+}
+
+void
+_test_frame_exit(struct trapframe *frame)
+{
+	thread_t td = curthread;
+
+	if (ISPL(frame->tf_cs) == SEL_UPL) {
+		KKASSERT(td->td_lwp);
+                KASSERT(td->td_lwp->lwp_md.md_regs == frame,
+                        ("_test_frame_exit: Frame mismatch %p %p",
+			td->td_lwp->lwp_md.md_regs, frame));
+		if (td->td_lwp->lwp_saveusp != (void *)frame->tf_rsp) {
+			kprintf("_test_frame_exit: %s:%d usp mismatch %p/%p\n",
+				td->td_comm, td->td_proc->p_pid,
+				td->td_lwp->lwp_saveusp,
+				(void *)frame->tf_rsp);
+		}
+		if (td->td_lwp->lwp_saveupc != (void *)frame->tf_rip) {
+			kprintf("_test_frame_exit: %s:%d upc mismatch %p/%p\n",
+				td->td_comm, td->td_proc->p_pid,
+				td->td_lwp->lwp_saveupc,
+				(void *)frame->tf_rip);
+		}
+
+		/*
+		 * adulterate the fields to catch entries that
+		 * don't run through test_frame_enter
+		 */
+		td->td_lwp->lwp_saveusp =
+			(void *)~(intptr_t)td->td_lwp->lwp_saveusp;
+		td->td_lwp->lwp_saveupc =
+			(void *)~(intptr_t)td->td_lwp->lwp_saveupc;
+	}
+	if ((char *)frame < td->td_kstack ||
+	    (char *)frame > td->td_kstack + td->td_kstack_size) {
+		panic("_test_frame_exit: frame not on kstack %p kstack=%p\n",
+			frame, td->td_kstack);
+	}
+}
+
+#endif
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 9ae50ce458..ee829b499e 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -223,6 +223,8 @@ struct lwp {
 	u_int		lwp_kqueue_serial;
 	struct lwkt_token lwp_token;	/* per-lwp token for signal/state */
 	struct spinlock lwp_spin;	/* spinlock for signal handling */
+	void		*lwp_reserveds1; /* reserved for lwp_saveusp */
+	void		*lwp_reserveds2; /* reserved for lwp_saveupc */
 };
 
 struct	proc {