[dragonfly.git] / sys / platform / pc64 / x86_64 / mplock.s

/*
 * $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
 * $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.21 2006/11/07 06:43:24 dillon Exp $
 *
 * Copyright (c) 2003,2004 The DragonFly Project.  All rights reserved.
 * 
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * 
 *				DragonFly MPLOCK operation
 *
 * Each thread has an MP lock count, td_mpcount, and there is a shared
 * global called mp_lock.  mp_lock is the physical MP lock and contains either
 * -1 or the cpuid of the cpu owning the lock.  The count is *NOT* integrated
 * into mp_lock but instead resides in each thread td_mpcount.
 *
 * When obtaining or releasing the MP lock the td_mpcount is PREDISPOSED
 * to the desired count *PRIOR* to operating on the mp_lock itself.  MP
 * lock operations can occur outside a critical section with interrupts
 * enabled with the provisio (which the routines below handle) that an
 * interrupt may come along and preempt us, racing our cmpxchgl instruction
 * to perform the operation we have requested by pre-disposing td_mpcount.
 *
 * Additionally, the LWKT threading system manages the MP lock and
 * lwkt_switch(), in particular, may be called after pre-disposing td_mpcount
 * to handle 'blocking' on the MP lock.
 *
 *
 * Recoded from the FreeBSD original:
 * ----------------------------------------------------------------------------
 * "THE BEER-WARE LICENSE" (Revision 42):
 * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
 * can do whatever you want with this stuff. If we meet some day, and you think
 * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
 * ----------------------------------------------------------------------------
 */

#include <machine/asmacros.h>
#include <machine_base/apic/apicreg.h>

#include "assym.s"

/*
 * YYY Debugging only.  Define this to be paranoid about invalidating the
 * TLB when we get giant.
 */
#undef PARANOID_INVLTLB

	.data
	ALIGN_DATA
#ifdef SMP
	.globl	mp_lock
mp_lock:
	.long	-1			/* initialized to not held */
#endif

	.text
	SUPERALIGN_TEXT

	/*
	 * Note on cmpxchgl... exchanges ecx with mem if mem matches eax.
	 * Z=1 (jz) on success.   A lock prefix is required for MP.
	 */
NON_GPROF_ENTRY(cpu_get_initial_mplock)
	movq	PCPU(curthread),%rcx
	movl	$1,TD_MPCOUNT(%rcx)	/* curthread has mpcount of 1 */
	movl	$0,mp_lock		/* owned by cpu 0 */
	NON_GPROF_RET

	/*
	 * cpu_try_mplock() returns non-zero on success, 0 on failure.  It
	 * only adjusts mp_lock, it does not touch td_mpcount.  Callers
	 * should always increment td_mpcount *before* trying to acquire
	 * the actual lock, predisposing td_mpcount to the desired state of
	 * the lock.
	 *
	 * NOTE! Only call cpu_try_mplock() inside a critical section.  If
	 * you don't an interrupt can come along and get and release
	 * the lock before our cmpxchgl instruction, causing us to fail 
	 * but resulting in the lock being held by our cpu.
	 */
NON_GPROF_ENTRY(cpu_try_mplock)
	movl	PCPU(cpuid),%ecx
	movl	$-1,%eax
	lock cmpxchgl %ecx,mp_lock	/* ecx<->mem if eax matches */
	jnz	1f
#ifdef PARANOID_INVLTLB
	movq	%cr3,%rax; movq %rax,%cr3	/* YYY check and remove */
#endif
	movl	$1,%eax
	NON_GPROF_RET
1:
	subl	%eax,%eax
	NON_GPROF_RET

	/*
	 * get_mplock() Obtains the MP lock and may switch away if it cannot
	 * get it.  This routine may be called WITHOUT a critical section
	 * and with cpu interrupts enabled.
	 *
	 * To handle races in a sane fashion we predispose TD_MPCOUNT,
	 * which prevents us from losing the lock in a race if we already
	 * have it or happen to get it.  It also means that we might get
	 * the lock in an interrupt race before we have a chance to execute
	 * our cmpxchgl instruction, so we have to handle that case.
	 * Fortunately simply calling lwkt_switch() handles the situation
	 * for us and also 'blocks' us until the MP lock can be obtained.
	 */
NON_GPROF_ENTRY(get_mplock)
	movl	PCPU(cpuid),%ecx
	movq	PCPU(curthread),%rdx
	incl	TD_MPCOUNT(%rdx)	/* predispose */
	cmpl	%ecx,mp_lock
	jne	1f
	NON_GPROF_RET			/* success! */

	/*
	 * We don't already own the mp_lock, use cmpxchgl to try to get
	 * it.
	 */
1:
	movl	$-1,%eax
	lock cmpxchgl %ecx,mp_lock
	jnz	2f
#ifdef PARANOID_INVLTLB
	movq	%cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
#endif
	NON_GPROF_RET			/* success */

	/*
	 * Failure, but we could end up owning mp_lock anyway due to
	 * an interrupt race.  lwkt_switch() will clean up the mess
	 * and 'block' until the mp_lock is obtained.
	 *
	 * Create a stack frame for the call so KTR logs the stack
	 * backtrace properly.
	 */
2:
	pushq	%rbp
	movq	%rsp,%rbp
	call	lwkt_mp_lock_contested
	popq	%rbp
#ifdef INVARIANTS
	movl	PCPU(cpuid),%eax	/* failure */
	cmpl	%eax,mp_lock
	jne	4f
#endif
	NON_GPROF_RET
#ifdef INVARIANTS
4:
	cmpl	$0,panicstr		/* don't double panic */
	je	badmp_get2
	NON_GPROF_RET
#endif

	/*
	 * try_mplock() attempts to obtain the MP lock.  1 is returned on
	 * success, 0 on failure.  We do not have to be in a critical section
	 * and interrupts are almost certainly enabled.
	 *
	 * We must pre-dispose TD_MPCOUNT in order to deal with races in
	 * a reasonable way.
	 *
	 */
NON_GPROF_ENTRY(try_mplock)
	movl	PCPU(cpuid),%ecx
	movq	PCPU(curthread),%rdx
	incl	TD_MPCOUNT(%rdx)		/* pre-dispose for race */
	cmpl	%ecx,mp_lock
	je	1f				/* trivial success */
	movl	$-1,%eax
	lock cmpxchgl %ecx,mp_lock
	jnz	2f
	/*
	 * Success
	 */
#ifdef PARANOID_INVLTLB
	movq	%cr3,%rax; movq %rax,%cr3	/* YYY check and remove */
#endif
1:
	movl	$1,%eax				/* success (cmpxchgl good!) */
	NON_GPROF_RET

	/*
	 * The cmpxchgl failed but we might have raced.  Undo the mess by
	 * predispoing TD_MPCOUNT and then checking.  If TD_MPCOUNT is
	 * still non-zero we don't care what state the lock is in (since
	 * we obviously didn't own it above), just return failure even if
	 * we won the lock in an interrupt race.  If TD_MPCOUNT is zero
	 * make sure we don't own the lock in case we did win it in a race.
	 */
2:
	decl	TD_MPCOUNT(%rdx)
	cmpl	$0,TD_MPCOUNT(%rdx)
	jne	3f
	movl	PCPU(cpuid),%eax
	movl	$-1,%ecx
	lock cmpxchgl %ecx,mp_lock
3:
	subl	%eax,%eax
	NON_GPROF_RET
	
	/*
	 * rel_mplock() releases a previously obtained MP lock.
	 *
	 * In order to release the MP lock we pre-dispose TD_MPCOUNT for
	 * the release and basically repeat the release portion of try_mplock
	 * above.
	 */
NON_GPROF_ENTRY(rel_mplock)
	movq	PCPU(curthread),%rdx
	movl	TD_MPCOUNT(%rdx),%eax
#ifdef INVARIANTS
	cmpl	$0,%eax
	je	badmp_rel
#endif
	subl	$1,%eax
	movl	%eax,TD_MPCOUNT(%rdx)
	cmpl	$0,%eax
	jne	3f
	movl	PCPU(cpuid),%eax
	movl	$-1,%ecx
	lock cmpxchgl %ecx,mp_lock
	movl	mp_lock_contention_mask,%eax
	cmpl	$0,%eax
	je	3f
	call	lwkt_mp_lock_uncontested
3:
	NON_GPROF_RET

#ifdef INVARIANTS

badmp_get:
	movq	$bmpsw1,%rdi
	movl	$0,%eax
	call	panic
badmp_get2:
	movq	$bmpsw1a,%rdi
	movl	$0,%eax
	call	panic
badmp_rel:
	movq	$bmpsw2,%rdi
	movl	$0,%eax
	call	panic

	.data

bmpsw1:
	.asciz	"try/get_mplock(): already have lock! %d %p"

bmpsw1a:
	.asciz	"try/get_mplock(): failed on count or switch %d %p"

bmpsw2:
	.asciz	"rel_mplock(): mpcount already 0 @ %p %p %p %p %p %p %p %p!"

#endif
Commit	Line	Data
d7f50089 YY	1	/*
d7f50089 YY	2	* $FreeBSD: src/sys/i386/i386/mplock.s,v 1.29.2.2 2000/05/16 06:58:06 dillon Exp $
46d4e165	3	* $DragonFly: src/sys/platform/pc32/i386/mplock.s,v 1.21 2006/11/07 06:43:24 dillon Exp $
d7f50089 YY	4	*
	5	* Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
	6	*
	7	* This code is derived from software contributed to The DragonFly Project
	8	* by Matthew Dillon <dillon@backplane.com>
	9	*
	10	* Redistribution and use in source and binary forms, with or without
	11	* modification, are permitted provided that the following conditions
	12	* are met:
	13	*
	14	* 1. Redistributions of source code must retain the above copyright
	15	* notice, this list of conditions and the following disclaimer.
	16	* 2. Redistributions in binary form must reproduce the above copyright
	17	* notice, this list of conditions and the following disclaimer in
	18	* the documentation and/or other materials provided with the
	19	* distribution.
	20	* 3. Neither the name of The DragonFly Project nor the names of its
	21	* contributors may be used to endorse or promote products derived
	22	* from this software without specific, prior written permission.
	23	*
	24	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	25	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	26	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	27	* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	28	* COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	29	* INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
	30	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	31	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	32	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	33	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
	34	* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	35	* SUCH DAMAGE.
	36	*
	37	* DragonFly MPLOCK operation
	38	*
	39	* Each thread has an MP lock count, td_mpcount, and there is a shared
	40	* global called mp_lock. mp_lock is the physical MP lock and contains either
	41	* -1 or the cpuid of the cpu owning the lock. The count is NOT integrated
	42	* into mp_lock but instead resides in each thread td_mpcount.
	43	*
	44	* When obtaining or releasing the MP lock the td_mpcount is PREDISPOSED
	45	* to the desired count PRIOR to operating on the mp_lock itself. MP
	46	* lock operations can occur outside a critical section with interrupts
	47	* enabled with the provisio (which the routines below handle) that an
	48	* interrupt may come along and preempt us, racing our cmpxchgl instruction
	49	* to perform the operation we have requested by pre-disposing td_mpcount.
	50	*
	51	* Additionally, the LWKT threading system manages the MP lock and
	52	* lwkt_switch(), in particular, may be called after pre-disposing td_mpcount
	53	* to handle 'blocking' on the MP lock.
	54	*
	55	*
	56	* Recoded from the FreeBSD original:
	57	* ----------------------------------------------------------------------------
	58	* "THE BEER-WARE LICENSE" (Revision 42):
	59	* <phk@FreeBSD.org> wrote this file. As long as you retain this notice you
	60	* can do whatever you want with this stuff. If we meet some day, and you think
	61	* this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
	62	* ----------------------------------------------------------------------------
	63	*/
	64
	65	#include <machine/asmacros.h>
d7f50089	66	#include <machine_base/apic/apicreg.h>
d7f50089 YY	67
	68	#include "assym.s"
	69
	70	/*
	71	* YYY Debugging only. Define this to be paranoid about invalidating the
	72	* TLB when we get giant.
	73	*/
	74	#undef PARANOID_INVLTLB
	75
	76	.data
	77	ALIGN_DATA
	78	#ifdef SMP
	79	.globl mp_lock
	80	mp_lock:
	81	.long -1 /* initialized to not held */
	82	#endif
	83
	84	.text
	85	SUPERALIGN_TEXT
	86
	87	/*
	88	* Note on cmpxchgl... exchanges ecx with mem if mem matches eax.
	89	* Z=1 (jz) on success. A lock prefix is required for MP.
	90	*/
	91	NON_GPROF_ENTRY(cpu_get_initial_mplock)
46d4e165 JG	92	movq PCPU(curthread),%rcx
46d4e165 JG	93	movl $1,TD_MPCOUNT(%rcx) /* curthread has mpcount of 1 */
d7f50089 YY	94	movl $0,mp_lock /* owned by cpu 0 */
	95	NON_GPROF_RET
	96
	97	/*
	98	* cpu_try_mplock() returns non-zero on success, 0 on failure. It
	99	* only adjusts mp_lock, it does not touch td_mpcount. Callers
	100	* should always increment td_mpcount before trying to acquire
	101	* the actual lock, predisposing td_mpcount to the desired state of
	102	* the lock.
	103	*
	104	* NOTE! Only call cpu_try_mplock() inside a critical section. If
	105	* you don't an interrupt can come along and get and release
	106	* the lock before our cmpxchgl instruction, causing us to fail
	107	* but resulting in the lock being held by our cpu.
	108	*/
	109	NON_GPROF_ENTRY(cpu_try_mplock)
	110	movl PCPU(cpuid),%ecx
	111	movl $-1,%eax
	112	lock cmpxchgl %ecx,mp_lock /* ecx<->mem if eax matches */
	113	jnz 1f
	114	#ifdef PARANOID_INVLTLB
46d4e165	115	movq %cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
d7f50089 YY	116	#endif
	117	movl $1,%eax
	118	NON_GPROF_RET
	119	1:
	120	subl %eax,%eax
	121	NON_GPROF_RET
	122
	123	/*
	124	* get_mplock() Obtains the MP lock and may switch away if it cannot
	125	* get it. This routine may be called WITHOUT a critical section
	126	* and with cpu interrupts enabled.
	127	*
	128	* To handle races in a sane fashion we predispose TD_MPCOUNT,
	129	* which prevents us from losing the lock in a race if we already
	130	* have it or happen to get it. It also means that we might get
	131	* the lock in an interrupt race before we have a chance to execute
	132	* our cmpxchgl instruction, so we have to handle that case.
	133	* Fortunately simply calling lwkt_switch() handles the situation
	134	* for us and also 'blocks' us until the MP lock can be obtained.
	135	*/
	136	NON_GPROF_ENTRY(get_mplock)
	137	movl PCPU(cpuid),%ecx
46d4e165 JG	138	movq PCPU(curthread),%rdx
46d4e165 JG	139	incl TD_MPCOUNT(%rdx) /* predispose */
d7f50089 YY	140	cmpl %ecx,mp_lock
	141	jne 1f
	142	NON_GPROF_RET /* success! */
	143
	144	/*
	145	* We don't already own the mp_lock, use cmpxchgl to try to get
	146	* it.
	147	*/
	148	1:
	149	movl $-1,%eax
	150	lock cmpxchgl %ecx,mp_lock
	151	jnz 2f
46d4e165 JG	152	#ifdef PARANOID_INVLTLB
	153	movq %cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
	154	#endif
d7f50089 YY	155	NON_GPROF_RET /* success */
	156
	157	/*
	158	* Failure, but we could end up owning mp_lock anyway due to
	159	* an interrupt race. lwkt_switch() will clean up the mess
	160	* and 'block' until the mp_lock is obtained.
	161	*
	162	* Create a stack frame for the call so KTR logs the stack
	163	* backtrace properly.
	164	*/
	165	2:
46d4e165 JG	166	pushq %rbp
46d4e165 JG	167	movq %rsp,%rbp
d7f50089	168	call lwkt_mp_lock_contested
46d4e165	169	popq %rbp
d7f50089 YY	170	#ifdef INVARIANTS
	171	movl PCPU(cpuid),%eax /* failure */
	172	cmpl %eax,mp_lock
	173	jne 4f
	174	#endif
	175	NON_GPROF_RET
	176	#ifdef INVARIANTS
	177	4:
	178	cmpl $0,panicstr /* don't double panic */
	179	je badmp_get2
	180	NON_GPROF_RET
	181	#endif
	182
	183	/*
	184	* try_mplock() attempts to obtain the MP lock. 1 is returned on
	185	* success, 0 on failure. We do not have to be in a critical section
	186	* and interrupts are almost certainly enabled.
	187	*
	188	* We must pre-dispose TD_MPCOUNT in order to deal with races in
	189	* a reasonable way.
	190	*
	191	*/
	192	NON_GPROF_ENTRY(try_mplock)
	193	movl PCPU(cpuid),%ecx
46d4e165 JG	194	movq PCPU(curthread),%rdx
46d4e165 JG	195	incl TD_MPCOUNT(%rdx) /* pre-dispose for race */
d7f50089 YY	196	cmpl %ecx,mp_lock
	197	je 1f /* trivial success */
	198	movl $-1,%eax
	199	lock cmpxchgl %ecx,mp_lock
	200	jnz 2f
	201	/*
	202	* Success
	203	*/
	204	#ifdef PARANOID_INVLTLB
46d4e165	205	movq %cr3,%rax; movq %rax,%cr3 /* YYY check and remove */
d7f50089 YY	206	#endif
	207	1:
	208	movl $1,%eax /* success (cmpxchgl good!) */
	209	NON_GPROF_RET
	210
	211	/*
	212	* The cmpxchgl failed but we might have raced. Undo the mess by
	213	* predispoing TD_MPCOUNT and then checking. If TD_MPCOUNT is
	214	* still non-zero we don't care what state the lock is in (since
	215	* we obviously didn't own it above), just return failure even if
	216	* we won the lock in an interrupt race. If TD_MPCOUNT is zero
	217	* make sure we don't own the lock in case we did win it in a race.
	218	*/
	219	2:
46d4e165 JG	220	decl TD_MPCOUNT(%rdx)
46d4e165 JG	221	cmpl $0,TD_MPCOUNT(%rdx)
d7f50089 YY	222	jne 3f
	223	movl PCPU(cpuid),%eax
	224	movl $-1,%ecx
	225	lock cmpxchgl %ecx,mp_lock
	226	3:
	227	subl %eax,%eax
	228	NON_GPROF_RET
	229
	230	/*
	231	* rel_mplock() releases a previously obtained MP lock.
	232	*
	233	* In order to release the MP lock we pre-dispose TD_MPCOUNT for
	234	* the release and basically repeat the release portion of try_mplock
	235	* above.
	236	*/
	237	NON_GPROF_ENTRY(rel_mplock)
46d4e165 JG	238	movq PCPU(curthread),%rdx
46d4e165 JG	239	movl TD_MPCOUNT(%rdx),%eax
d7f50089 YY	240	#ifdef INVARIANTS
	241	cmpl $0,%eax
	242	je badmp_rel
	243	#endif
	244	subl $1,%eax
46d4e165	245	movl %eax,TD_MPCOUNT(%rdx)
d7f50089 YY	246	cmpl $0,%eax
	247	jne 3f
	248	movl PCPU(cpuid),%eax
	249	movl $-1,%ecx
	250	lock cmpxchgl %ecx,mp_lock
46d4e165 JG	251	movl mp_lock_contention_mask,%eax
	252	cmpl $0,%eax
	253	je 3f
	254	call lwkt_mp_lock_uncontested
d7f50089 YY	255	3:
	256	NON_GPROF_RET
	257
	258	#ifdef INVARIANTS
	259
	260	badmp_get:
46d4e165 JG	261	movq $bmpsw1,%rdi
46d4e165 JG	262	movl $0,%eax
d7f50089 YY	263	call panic
d7f50089 YY	264	badmp_get2:
46d4e165 JG	265	movq $bmpsw1a,%rdi
46d4e165 JG	266	movl $0,%eax
d7f50089 YY	267	call panic
d7f50089 YY	268	badmp_rel:
46d4e165 JG	269	movq $bmpsw2,%rdi
46d4e165 JG	270	movl $0,%eax
d7f50089 YY	271	call panic
	272
	273	.data
	274
	275	bmpsw1:
	276	.asciz "try/get_mplock(): already have lock! %d %p"
	277
	278	bmpsw1a:
	279	.asciz "try/get_mplock(): failed on count or switch %d %p"
	280
	281	bmpsw2:
	282	.asciz "rel_mplock(): mpcount already 0 @ %p %p %p %p %p %p %p %p!"
	283
	284	#endif
	285