1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
|
/* rwsem.S: RW semaphore assembler.
*
* Written by David S. Miller (davem@redhat.com), 2001.
* Derived from asm-i386/rwsem.h
*/
#include <asm/rwsem-const.h>
.section .sched.text, "ax"
.globl __down_read
__down_read:
1: lduw [%o0], %g1
add %g1, 1, %g7
cas [%o0], %g1, %g7
cmp %g1, %g7
bne,pn %icc, 1b
add %g7, 1, %g7
cmp %g7, 0
membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
nop
2:
retl
nop
3:
save %sp, -192, %sp
call rwsem_down_read_failed
mov %i0, %o0
ret
restore
.size __down_read, .-__down_read
.globl __down_read_trylock
__down_read_trylock:
1: lduw [%o0], %g1
add %g1, 1, %g7
cmp %g7, 0
bl,pn %icc, 2f
mov 0, %o1
cas [%o0], %g1, %g7
cmp %g1, %g7
bne,pn %icc, 1b
mov 1, %o1
membar #StoreLoad | #StoreStore
2: retl
mov %o1, %o0
.size __down_read_trylock, .-__down_read_trylock
.globl __down_write
__down_write:
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
1:
lduw [%o0], %g3
add %g3, %g1, %g7
cas [%o0], %g3, %g7
cmp %g3, %g7
bne,pn %icc, 1b
cmp %g7, 0
membar #StoreLoad | #StoreStore
bne,pn %icc, 3f
nop
2: retl
nop
3:
save %sp, -192, %sp
call rwsem_down_write_failed
mov %i0, %o0
ret
restore
.size __down_write, .-__down_write
.globl __down_write_trylock
__down_write_trylock:
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
1:
lduw [%o0], %g3
cmp %g3, 0
bne,pn %icc, 2f
mov 0, %o1
add %g3, %g1, %g7
cas [%o0], %g3, %g7
cmp %g3, %g7
bne,pn %icc, 1b
mov 1, %o1
membar #StoreLoad | #StoreStore
2: retl
mov %o1, %o0
.size __down_write_trylock, .-__down_write_trylock
.globl __up_read
__up_read:
1:
lduw [%o0], %g1
sub %g1, 1, %g7
cas [%o0], %g1, %g7
cmp %g1, %g7
bne,pn %icc, 1b
cmp %g7, 0
membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
nop
2: retl
nop
3: sethi %hi(RWSEM_ACTIVE_MASK), %g1
sub %g7, 1, %g7
or %g1, %lo(RWSEM_ACTIVE_MASK), %g1
andcc %g7, %g1, %g0
bne,pn %icc, 2b
nop
save %sp, -192, %sp
call rwsem_wake
mov %i0, %o0
ret
restore
.size __up_read, .-__up_read
.globl __up_write
__up_write:
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
1:
lduw [%o0], %g3
sub %g3, %g1, %g7
cas [%o0], %g3, %g7
cmp %g3, %g7
bne,pn %icc, 1b
sub %g7, %g1, %g7
cmp %g7, 0
membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
nop
2:
retl
nop
3:
save %sp, -192, %sp
call rwsem_wake
mov %i0, %o0
ret
restore
.size __up_write, .-__up_write
.globl __downgrade_write
__downgrade_write:
sethi %hi(RWSEM_WAITING_BIAS), %g1
or %g1, %lo(RWSEM_WAITING_BIAS), %g1
1:
lduw [%o0], %g3
sub %g3, %g1, %g7
cas [%o0], %g3, %g7
cmp %g3, %g7
bne,pn %icc, 1b
sub %g7, %g1, %g7
cmp %g7, 0
membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
nop
2:
retl
nop
3:
save %sp, -192, %sp
call rwsem_downgrade_wake
mov %i0, %o0
ret
restore
.size __downgrade_write, .-__downgrade_write
|
