1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2024, Google LLC.
* Pasha Tatashin <pasha.tatashin@soleen.com>
*/
#ifndef __IOMMU_PAGES_H
#define __IOMMU_PAGES_H
#include <linux/vmstat.h>
#include <linux/gfp.h>
#include <linux/mm.h>
/*
* All page allocations that should be reported to as "iommu-pagetables" to
* userspace must use one of the functions below. This includes allocations of
* page-tables and other per-iommu_domain configuration structures.
*
* This is necessary for the proper accounting as IOMMU state can be rather
* large, i.e. multiple gigabytes in size.
*/
/**
* __iommu_alloc_account - account for newly allocated page.
* @page: head struct page of the page.
* @order: order of the page
*/
static inline void __iommu_alloc_account(struct page *page, int order)
{
const long pgcnt = 1l << order;
mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);
mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);
}
/**
* __iommu_free_account - account a page that is about to be freed.
* @page: head struct page of the page.
* @order: order of the page
*/
static inline void __iommu_free_account(struct page *page, int order)
{
const long pgcnt = 1l << order;
mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);
mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);
}
/**
* __iommu_alloc_pages - allocate a zeroed page of a given order.
* @gfp: buddy allocator flags
* @order: page order
*
* returns the head struct page of the allocated page.
*/
static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order)
{
struct page *page;
page = alloc_pages(gfp | __GFP_ZERO, order);
if (unlikely(!page))
return NULL;
__iommu_alloc_account(page, order);
return page;
}
/**
* __iommu_free_pages - free page of a given order
* @page: head struct page of the page
* @order: page order
*/
static inline void __iommu_free_pages(struct page *page, int order)
{
if (!page)
return;
__iommu_free_account(page, order);
__free_pages(page, order);
}
/**
* iommu_alloc_pages_node - allocate a zeroed page of a given order from
* specific NUMA node.
* @nid: memory NUMA node id
* @gfp: buddy allocator flags
* @order: page order
*
* returns the virtual address of the allocated page
*/
static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order)
{
struct page *page = alloc_pages_node(nid, gfp | __GFP_ZERO, order);
if (unlikely(!page))
return NULL;
__iommu_alloc_account(page, order);
return page_address(page);
}
/**
* iommu_alloc_pages - allocate a zeroed page of a given order
* @gfp: buddy allocator flags
* @order: page order
*
* returns the virtual address of the allocated page
*/
static inline void *iommu_alloc_pages(gfp_t gfp, int order)
{
struct page *page = __iommu_alloc_pages(gfp, order);
if (unlikely(!page))
return NULL;
return page_address(page);
}
/**
* iommu_alloc_page_node - allocate a zeroed page at specific NUMA node.
* @nid: memory NUMA node id
* @gfp: buddy allocator flags
*
* returns the virtual address of the allocated page
*/
static inline void *iommu_alloc_page_node(int nid, gfp_t gfp)
{
return iommu_alloc_pages_node(nid, gfp, 0);
}
/**
* iommu_alloc_page - allocate a zeroed page
* @gfp: buddy allocator flags
*
* returns the virtual address of the allocated page
*/
static inline void *iommu_alloc_page(gfp_t gfp)
{
return iommu_alloc_pages(gfp, 0);
}
/**
* iommu_free_pages - free page of a given order
* @virt: virtual address of the page to be freed.
* @order: page order
*/
static inline void iommu_free_pages(void *virt, int order)
{
if (!virt)
return;
__iommu_free_pages(virt_to_page(virt), order);
}
/**
* iommu_free_page - free page
* @virt: virtual address of the page to be freed.
*/
static inline void iommu_free_page(void *virt)
{
iommu_free_pages(virt, 0);
}
/**
* iommu_put_pages_list - free a list of pages.
* @page: the head of the lru list to be freed.
*
* There are no locking requirement for these pages, as they are going to be
* put on a free list as soon as refcount reaches 0. Pages are put on this LRU
* list once they are removed from the IOMMU page tables. However, they can
* still be access through debugfs.
*/
static inline void iommu_put_pages_list(struct list_head *page)
{
while (!list_empty(page)) {
struct page *p = list_entry(page->prev, struct page, lru);
list_del(&p->lru);
__iommu_free_account(p, 0);
put_page(p);
}
}
#endif /* __IOMMU_PAGES_H */
|
