1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
// SPDX-License-Identifier: GPL-2.0
/*
* jump label x86 support
*
* Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
*
*/
#include <linux/jump_label.h>
#include <linux/memory.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/cpu.h>
#include <asm/kprobes.h>
#include <asm/alternative.h>
#include <asm/text-patching.h>
#include <asm/insn.h>
int arch_jump_entry_size(struct jump_entry *entry)
{
struct insn insn = {};
insn_decode_kernel(&insn, (void *)jump_entry_code(entry));
BUG_ON(insn.length != 2 && insn.length != 5);
return insn.length;
}
struct jump_label_patch {
const void *code;
int size;
};
static struct jump_label_patch
__jump_label_patch(struct jump_entry *entry, enum jump_label_type type)
{
const void *expect, *code, *nop;
const void *addr, *dest;
int size;
addr = (void *)jump_entry_code(entry);
dest = (void *)jump_entry_target(entry);
size = arch_jump_entry_size(entry);
switch (size) {
case JMP8_INSN_SIZE:
code = text_gen_insn(JMP8_INSN_OPCODE, addr, dest);
nop = x86_nops[size];
break;
case JMP32_INSN_SIZE:
code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
nop = x86_nops[size];
break;
default: BUG();
}
if (type == JUMP_LABEL_JMP)
expect = nop;
else
expect = code;
if (memcmp(addr, expect, size)) {
/*
* The location is not an op that we were expecting.
* Something went wrong. Crash the box, as something could be
* corrupting the kernel.
*/
pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) size:%d type:%d\n",
addr, addr, addr, expect, size, type);
BUG();
}
if (type == JUMP_LABEL_NOP)
code = nop;
return (struct jump_label_patch){.code = code, .size = size};
}
static __always_inline void
__jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
int init)
{
const struct jump_label_patch jlp = __jump_label_patch(entry, type);
/*
* As long as only a single processor is running and the code is still
* not marked as RO, text_poke_early() can be used; Checking that
* system_state is SYSTEM_BOOTING guarantees it. It will be set to
* SYSTEM_SCHEDULING before other cores are awaken and before the
* code is write-protected.
*
* At the time the change is being done, just ignore whether we
* are doing nop -> jump or jump -> nop transition, and assume
* always nop being the 'currently valid' instruction
*/
if (init || system_state == SYSTEM_BOOTING) {
text_poke_early((void *)jump_entry_code(entry), jlp.code, jlp.size);
return;
}
text_poke_bp((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
}
static void __ref jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
int init)
{
mutex_lock(&text_mutex);
__jump_label_transform(entry, type, init);
mutex_unlock(&text_mutex);
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
jump_label_transform(entry, type, 0);
}
bool arch_jump_label_transform_queue(struct jump_entry *entry,
enum jump_label_type type)
{
struct jump_label_patch jlp;
if (system_state == SYSTEM_BOOTING) {
/*
* Fallback to the non-batching mode.
*/
arch_jump_label_transform(entry, type);
return true;
}
mutex_lock(&text_mutex);
jlp = __jump_label_patch(entry, type);
text_poke_queue((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
mutex_unlock(&text_mutex);
return true;
}
void arch_jump_label_transform_apply(void)
{
mutex_lock(&text_mutex);
text_poke_finish();
mutex_unlock(&text_mutex);
}
static enum {
JL_STATE_START,
JL_STATE_NO_UPDATE,
JL_STATE_UPDATE,
} jlstate __initdata_or_module = JL_STATE_START;
__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
if (jlstate == JL_STATE_UPDATE)
jump_label_transform(entry, type, 1);
}
|
