summaryrefslogtreecommitdiff
path: root/arch/arm64/kernel/compat_alignment.c
blob: deff21bfa6800cfc2ceefd36c6e06f01880a1f4f (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
// SPDX-License-Identifier: GPL-2.0-only
// based on arch/arm/mm/alignment.c

#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/perf_event.h>
#include <linux/uaccess.h>

#include <asm/exception.h>
#include <asm/ptrace.h>
#include <asm/traps.h>

/*
 * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998
 *
 * Speed optimisations and better fault handling by Russell King.
 */
#define CODING_BITS(i)	(i & 0x0e000000)

#define LDST_P_BIT(i)	(i & (1 << 24))		/* Preindex		*/
#define LDST_U_BIT(i)	(i & (1 << 23))		/* Add offset		*/
#define LDST_W_BIT(i)	(i & (1 << 21))		/* Writeback		*/
#define LDST_L_BIT(i)	(i & (1 << 20))		/* Load			*/

#define LDST_P_EQ_U(i)	((((i) ^ ((i) >> 1)) & (1 << 23)) == 0)

#define LDSTHD_I_BIT(i)	(i & (1 << 22))		/* double/half-word immed */

#define RN_BITS(i)	((i >> 16) & 15)	/* Rn			*/
#define RD_BITS(i)	((i >> 12) & 15)	/* Rd			*/
#define RM_BITS(i)	(i & 15)		/* Rm			*/

#define REGMASK_BITS(i)	(i & 0xffff)

#define BAD_INSTR 	0xdeadc0de

/* Thumb-2 32 bit format per ARMv7 DDI0406A A6.3, either f800h,e800h,f800h */
#define IS_T32(hi16) \
	(((hi16) & 0xe000) == 0xe000 && ((hi16) & 0x1800))

union offset_union {
	unsigned long un;
	  signed long sn;
};

#define TYPE_ERROR	0
#define TYPE_FAULT	1
#define TYPE_LDST	2
#define TYPE_DONE	3

static void
do_alignment_finish_ldst(unsigned long addr, u32 instr, struct pt_regs *regs,
			 union offset_union offset)
{
	if (!LDST_U_BIT(instr))
		offset.un = -offset.un;

	if (!LDST_P_BIT(instr))
		addr += offset.un;

	if (!LDST_P_BIT(instr) || LDST_W_BIT(instr))
		regs->regs[RN_BITS(instr)] = addr;
}

static int
do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs)
{
	unsigned int rd = RD_BITS(instr);
	unsigned int rd2;
	int load;

	if ((instr & 0xfe000000) == 0xe8000000) {
		/* ARMv7 Thumb-2 32-bit LDRD/STRD */
		rd2 = (instr >> 8) & 0xf;
		load = !!(LDST_L_BIT(instr));
	} else if (((rd & 1) == 1) || (rd == 14)) {
		return TYPE_ERROR;
	} else {
		load = ((instr & 0xf0) == 0xd0);
		rd2 = rd + 1;
	}

	if (load) {
		unsigned int val, val2;

		if (get_user(val, (u32 __user *)addr) ||
		    get_user(val2, (u32 __user *)(addr + 4)))
			return TYPE_FAULT;
		regs->regs[rd] = val;
		regs->regs[rd2] = val2;
	} else {
		if (put_user(regs->regs[rd], (u32 __user *)addr) ||
		    put_user(regs->regs[rd2], (u32 __user *)(addr + 4)))
			return TYPE_FAULT;
	}
	return TYPE_LDST;
}

/*
 * LDM/STM alignment handler.
 *
 * There are 4 variants of this instruction:
 *
 * B = rn pointer before instruction, A = rn pointer after instruction
 *              ------ increasing address ----->
 *	        |    | r0 | r1 | ... | rx |    |
 * PU = 01             B                    A
 * PU = 11        B                    A
 * PU = 00        A                    B
 * PU = 10             A                    B
 */
static int
do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs)
{
	unsigned int rd, rn, nr_regs, regbits;
	unsigned long eaddr, newaddr;
	unsigned int val;

	/* count the number of registers in the mask to be transferred */
	nr_regs = hweight16(REGMASK_BITS(instr)) * 4;

	rn = RN_BITS(instr);
	newaddr = eaddr = regs->regs[rn];

	if (!LDST_U_BIT(instr))
		nr_regs = -nr_regs;
	newaddr += nr_regs;
	if (!LDST_U_BIT(instr))
		eaddr = newaddr;

	if (LDST_P_EQ_U(instr))	/* U = P */
		eaddr += 4;

	for (regbits = REGMASK_BITS(instr), rd = 0; regbits;
	     regbits >>= 1, rd += 1)
		if (regbits & 1) {
			if (LDST_L_BIT(instr)) {
				if (get_user(val, (u32 __user *)eaddr))
					return TYPE_FAULT;
				if (rd < 15)
					regs->regs[rd] = val;
				else
					regs->pc = val;
			} else {
				/*
				 * The PC register has a bias of +8 in ARM mode
				 * and +4 in Thumb mode. This means that a read
				 * of the value of PC should account for this.
				 * Since Thumb does not permit STM instructions
				 * to refer to PC, just add 8 here.
				 */
				val = (rd < 15) ? regs->regs[rd] : regs->pc + 8;
				if (put_user(val, (u32 __user *)eaddr))
					return TYPE_FAULT;
			}
			eaddr += 4;
		}

	if (LDST_W_BIT(instr))
		regs->regs[rn] = newaddr;

	return TYPE_DONE;
}

/*
 * Convert Thumb multi-word load/store instruction forms to equivalent ARM
 * instructions so we can reuse ARM userland alignment fault fixups for Thumb.
 *
 * This implementation was initially based on the algorithm found in
 * gdb/sim/arm/thumbemu.c. It is basically just a code reduction of same
 * to convert only Thumb ld/st instruction forms to equivalent ARM forms.
 *
 * NOTES:
 * 1. Comments below refer to ARM ARM DDI0100E Thumb Instruction sections.
 * 2. If for some reason we're passed an non-ld/st Thumb instruction to
 *    decode, we return 0xdeadc0de. This should never happen under normal
 *    circumstances but if it does, we've got other problems to deal with
 *    elsewhere and we obviously can't fix those problems here.
 */

static unsigned long thumb2arm(u16 tinstr)
{
	u32 L = (tinstr & (1<<11)) >> 11;

	switch ((tinstr & 0xf800) >> 11) {
	/* 6.6.1 Format 1: */
	case 0xc000 >> 11:				/* 7.1.51 STMIA */
	case 0xc800 >> 11:				/* 7.1.25 LDMIA */
		{
			u32 Rn = (tinstr & (7<<8)) >> 8;
			u32 W = ((L<<Rn) & (tinstr&255)) ? 0 : 1<<21;

			return 0xe8800000 | W | (L<<20) | (Rn<<16) |
				(tinstr&255);
		}

	/* 6.6.1 Format 2: */
	case 0xb000 >> 11:				/* 7.1.48 PUSH */
	case 0xb800 >> 11:				/* 7.1.47 POP */
		if ((tinstr & (3 << 9)) == 0x0400) {
			static const u32 subset[4] = {
				0xe92d0000,	/* STMDB sp!,{registers} */
				0xe92d4000,	/* STMDB sp!,{registers,lr} */
				0xe8bd0000,	/* LDMIA sp!,{registers} */
				0xe8bd8000	/* LDMIA sp!,{registers,pc} */
			};
			return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] |
			    (tinstr & 255);		/* register_list */
		}
		fallthrough;	/* for illegal instruction case */

	default:
		return BAD_INSTR;
	}
}

/*
 * Convert Thumb-2 32 bit LDM, STM, LDRD, STRD to equivalent instruction
 * handlable by ARM alignment handler, also find the corresponding handler,
 * so that we can reuse ARM userland alignment fault fixups for Thumb.
 *
 * @pinstr: original Thumb-2 instruction; returns new handlable instruction
 * @regs: register context.
 * @poffset: return offset from faulted addr for later writeback
 *
 * NOTES:
 * 1. Comments below refer to ARMv7 DDI0406A Thumb Instruction sections.
 * 2. Register name Rt from ARMv7 is same as Rd from ARMv6 (Rd is Rt)
 */
static void *
do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs,
			    union offset_union *poffset)
{
	u32 instr = *pinstr;
	u16 tinst1 = (instr >> 16) & 0xffff;
	u16 tinst2 = instr & 0xffff;

	switch (tinst1 & 0xffe0) {
	/* A6.3.5 Load/Store multiple */
	case 0xe880:		/* STM/STMIA/STMEA,LDM/LDMIA, PUSH/POP T2 */
	case 0xe8a0:		/* ...above writeback version */
	case 0xe900:		/* STMDB/STMFD, LDMDB/LDMEA */
	case 0xe920:		/* ...above writeback version */
		/* no need offset decision since handler calculates it */
		return do_alignment_ldmstm;

	case 0xf840:		/* POP/PUSH T3 (single register) */
		if (RN_BITS(instr) == 13 && (tinst2 & 0x09ff) == 0x0904) {
			u32 L = !!(LDST_L_BIT(instr));
			const u32 subset[2] = {
				0xe92d0000,	/* STMDB sp!,{registers} */
				0xe8bd0000,	/* LDMIA sp!,{registers} */
			};
			*pinstr = subset[L] | (1<<RD_BITS(instr));
			return do_alignment_ldmstm;
		}
		/* Else fall through for illegal instruction case */
		break;

	/* A6.3.6 Load/store double, STRD/LDRD(immed, lit, reg) */
	case 0xe860:
	case 0xe960:
	case 0xe8e0:
	case 0xe9e0:
		poffset->un = (tinst2 & 0xff) << 2;
		fallthrough;

	case 0xe940:
	case 0xe9c0:
		return do_alignment_ldrdstrd;

	/*
	 * No need to handle load/store instructions up to word size
	 * since ARMv6 and later CPUs can perform unaligned accesses.
	 */
	default:
		break;
	}
	return NULL;
}

static int alignment_get_arm(struct pt_regs *regs, __le32 __user *ip, u32 *inst)
{
	__le32 instr = 0;
	int fault;

	fault = get_user(instr, ip);
	if (fault)
		return fault;

	*inst = __le32_to_cpu(instr);
	return 0;
}

static int alignment_get_thumb(struct pt_regs *regs, __le16 __user *ip, u16 *inst)
{
	__le16 instr = 0;
	int fault;

	fault = get_user(instr, ip);
	if (fault)
		return fault;

	*inst = __le16_to_cpu(instr);
	return 0;
}

int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs)
{
	union offset_union offset;
	unsigned long instrptr;
	int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs);
	unsigned int type;
	u32 instr = 0;
	int isize = 4;
	int thumb2_32b = 0;

	instrptr = instruction_pointer(regs);

	if (compat_thumb_mode(regs)) {
		__le16 __user *ptr = (__le16 __user *)(instrptr & ~1);
		u16 tinstr, tinst2;

		if (alignment_get_thumb(regs, ptr, &tinstr))
			return 1;

		if (IS_T32(tinstr)) { /* Thumb-2 32-bit */
			if (alignment_get_thumb(regs, ptr + 1, &tinst2))
				return 1;
			instr = ((u32)tinstr << 16) | tinst2;
			thumb2_32b = 1;
		} else {
			isize = 2;
			instr = thumb2arm(tinstr);
		}
	} else {
		if (alignment_get_arm(regs, (__le32 __user *)instrptr, &instr))
			return 1;
	}

	switch (CODING_BITS(instr)) {
	case 0x00000000:	/* 3.13.4 load/store instruction extensions */
		if (LDSTHD_I_BIT(instr))
			offset.un = (instr & 0xf00) >> 4 | (instr & 15);
		else
			offset.un = regs->regs[RM_BITS(instr)];

		if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */
		    (instr & 0x001000f0) == 0x000000f0)   /* STRD */
			handler = do_alignment_ldrdstrd;
		else
			return 1;
		break;

	case 0x08000000:	/* ldm or stm, or thumb-2 32bit instruction */
		if (thumb2_32b) {
			offset.un = 0;
			handler = do_alignment_t32_to_handler(&instr, regs, &offset);
		} else {
			offset.un = 0;
			handler = do_alignment_ldmstm;
		}
		break;

	default:
		return 1;
	}

	type = handler(addr, instr, regs);

	if (type == TYPE_ERROR || type == TYPE_FAULT)
		return 1;

	if (type == TYPE_LDST)
		do_alignment_finish_ldst(addr, instr, regs, offset);

	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->pc);
	arm64_skip_faulting_instruction(regs, isize);

	return 0;
}