1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
/*
* In-kernel vector facility support functions
*
* Copyright IBM Corp. 2015
* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
*/
#include <linux/kernel.h>
#include <linux/cpu.h>
#include <linux/sched.h>
#include <asm/fpu/types.h>
#include <asm/fpu/api.h>
/*
* Per-CPU variable to maintain FPU register ranges that are in use
* by the kernel.
*/
static DEFINE_PER_CPU(u32, kernel_fpu_state);
#define KERNEL_FPU_STATE_MASK (KERNEL_FPU_MASK|KERNEL_FPC)
void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
{
if (!__this_cpu_read(kernel_fpu_state)) {
/*
* Save user space FPU state and register contents. Multiple
* calls because of interruptions do not matter and return
* immediately. This also sets CIF_FPU to lazy restore FP/VX
* register contents when returning to user space.
*/
save_fpu_regs();
}
/* Update flags to use the vector facility for KERNEL_FPR */
if (MACHINE_HAS_VX && (state->mask & KERNEL_FPR)) {
flags |= KERNEL_VXR_LOW | KERNEL_FPC;
flags &= ~KERNEL_FPR;
}
/* Save and update current kernel VX state */
state->mask = __this_cpu_read(kernel_fpu_state);
__this_cpu_or(kernel_fpu_state, flags & KERNEL_FPU_STATE_MASK);
/*
* If this is the first call to __kernel_fpu_begin(), no additional
* work is required.
*/
if (!(state->mask & KERNEL_FPU_STATE_MASK))
return;
/*
* If KERNEL_FPR is still set, the vector facility is not available
* and, thus, save floating-point control and registers only.
*/
if (state->mask & KERNEL_FPR) {
asm volatile("stfpc %0" : "=Q" (state->fpc));
asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
return;
}
/*
* If this is a nested call to __kernel_fpu_begin(), check the saved
* state mask to save and later restore the vector registers that
* are already in use. Let's start with checking floating-point
* controls.
*/
if (state->mask & KERNEL_FPC)
asm volatile("stfpc %0" : "=m" (state->fpc));
/* Test and save vector registers */
asm volatile (
/*
* Test if any vector register must be saved and, if so,
* test if all register can be saved.
*/
" tmll %[m],15\n" /* KERNEL_VXR_MASK */
" jz 20f\n" /* no work -> done */
" la 1,%[vxrs]\n" /* load save area */
" jo 18f\n" /* -> save V0..V31 */
/*
* Test if V8..V23 can be saved at once... this speeds up
* for KERNEL_fpu_MID only. Otherwise continue to split the
* range of vector registers into two halves and test them
* separately.
*/
" tmll %[m],6\n" /* KERNEL_VXR_MID */
" jo 17f\n" /* -> save V8..V23 */
/* Test and save the first half of 16 vector registers */
"1: tmll %[m],3\n" /* KERNEL_VXR_LOW */
" jz 10f\n" /* -> KERNEL_VXR_HIGH */
" jo 2f\n" /* 11 -> save V0..V15 */
" brc 4,3f\n" /* 01 -> save V0..V7 */
" brc 2,4f\n" /* 10 -> save V8..V15 */
/* Test and save the second half of 16 vector registers */
"10: tmll %[m],12\n" /* KERNEL_VXR_HIGH */
" jo 19f\n" /* 11 -> save V16..V31 */
" brc 4,11f\n" /* 01 -> save V16..V23 */
" brc 2,12f\n" /* 10 -> save V24..V31 */
" j 20f\n" /* 00 -> done */
/*
* Below are the vstm combinations to save multiple vector
* registers at once.
*/
"2: .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */
" j 10b\n" /* -> VXR_HIGH */
"3: .word 0xe707,0x1000,0x003e\n" /* vstm 0,7,0(1) */
" j 10b\n" /* -> VXR_HIGH */
"4: .word 0xe78f,0x1080,0x003e\n" /* vstm 8,15,128(1) */
" j 10b\n" /* -> VXR_HIGH */
"\n"
"11: .word 0xe707,0x1100,0x0c3e\n" /* vstm 16,23,256(1) */
" j 20f\n" /* -> done */
"12: .word 0xe78f,0x1180,0x0c3e\n" /* vstm 24,31,384(1) */
" j 20f\n" /* -> done */
"\n"
"17: .word 0xe787,0x1080,0x043e\n" /* vstm 8,23,128(1) */
" nill %[m],249\n" /* m &= ~VXR_MID */
" j 1b\n" /* -> VXR_LOW */
"\n"
"18: .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */
"19: .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */
"20:"
: [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
: [m] "d" (state->mask)
: "1", "cc");
}
EXPORT_SYMBOL(__kernel_fpu_begin);
void __kernel_fpu_end(struct kernel_fpu *state)
{
/* Just update the per-CPU state if there is nothing to restore */
if (!(state->mask & KERNEL_FPU_STATE_MASK))
goto update_fpu_state;
/*
* If KERNEL_FPR is specified, the vector facility is not available
* and, thus, restore floating-point control and registers only.
*/
if (state->mask & KERNEL_FPR) {
asm volatile("lfpc %0" : : "Q" (state->fpc));
asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
goto update_fpu_state;
}
/* Test and restore floating-point controls */
if (state->mask & KERNEL_FPC)
asm volatile("lfpc %0" : : "Q" (state->fpc));
/* Test and restore (load) vector registers */
asm volatile (
/*
* Test if any vector registers must be loaded and, if so,
* test if all registers can be loaded at once.
*/
" tmll %[m],15\n" /* KERNEL_VXR_MASK */
" jz 20f\n" /* no work -> done */
" la 1,%[vxrs]\n" /* load load area */
" jo 18f\n" /* -> load V0..V31 */
/*
* Test if V8..V23 can be restored at once... this speeds up
* for KERNEL_VXR_MID only. Otherwise continue to split the
* range of vector registers into two halves and test them
* separately.
*/
" tmll %[m],6\n" /* KERNEL_VXR_MID */
" jo 17f\n" /* -> load V8..V23 */
/* Test and load the first half of 16 vector registers */
"1: tmll %[m],3\n" /* KERNEL_VXR_LOW */
" jz 10f\n" /* -> KERNEL_VXR_HIGH */
" jo 2f\n" /* 11 -> load V0..V15 */
" brc 4,3f\n" /* 01 -> load V0..V7 */
" brc 2,4f\n" /* 10 -> load V8..V15 */
/* Test and load the second half of 16 vector registers */
"10: tmll %[m],12\n" /* KERNEL_VXR_HIGH */
" jo 19f\n" /* 11 -> load V16..V31 */
" brc 4,11f\n" /* 01 -> load V16..V23 */
" brc 2,12f\n" /* 10 -> load V24..V31 */
" j 20f\n" /* 00 -> done */
/*
* Below are the vstm combinations to load multiple vector
* registers at once.
*/
"2: .word 0xe70f,0x1000,0x0036\n" /* vlm 0,15,0(1) */
" j 10b\n" /* -> VXR_HIGH */
"3: .word 0xe707,0x1000,0x0036\n" /* vlm 0,7,0(1) */
" j 10b\n" /* -> VXR_HIGH */
"4: .word 0xe78f,0x1080,0x0036\n" /* vlm 8,15,128(1) */
" j 10b\n" /* -> VXR_HIGH */
"\n"
"11: .word 0xe707,0x1100,0x0c36\n" /* vlm 16,23,256(1) */
" j 20f\n" /* -> done */
"12: .word 0xe78f,0x1180,0x0c36\n" /* vlm 24,31,384(1) */
" j 20f\n" /* -> done */
"\n"
"17: .word 0xe787,0x1080,0x0436\n" /* vlm 8,23,128(1) */
" nill %[m],249\n" /* m &= ~VXR_MID */
" j 1b\n" /* -> VXR_LOW */
"\n"
"18: .word 0xe70f,0x1000,0x0036\n" /* vlm 0,15,0(1) */
"19: .word 0xe70f,0x1100,0x0c36\n" /* vlm 16,31,256(1) */
"20:"
:
: [vxrs] "Q" (*(struct vx_array *) &state->vxrs),
[m] "d" (state->mask)
: "1", "cc");
update_fpu_state:
/* Update current kernel VX state */
__this_cpu_write(kernel_fpu_state, state->mask);
}
EXPORT_SYMBOL(__kernel_fpu_end);
|
