blob: 2a3cd1b5efbb02a5f0ba2d2367be142981ed00bf (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
/* SPDX-License-Identifier: MIT */
/*
* Copyright © 2025 Intel Corporation
*/
#ifndef _XE_USERPTR_H_
#define _XE_USERPTR_H_
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/notifier.h>
#include <linux/scatterlist.h>
#include <linux/spinlock.h>
#include <drm/drm_gpusvm.h>
#include "xe_tlb_inval_types.h"
struct xe_vm;
struct xe_vma;
struct xe_userptr_vma;
/** struct xe_userptr_vm - User pointer VM level state */
struct xe_userptr_vm {
/**
* @userptr.repin_list: list of VMAs which are user pointers,
* and needs repinning. Protected by @lock.
*/
struct list_head repin_list;
/**
* @userptr.invalidated_lock: Protects the
* @userptr.invalidated list.
*/
spinlock_t invalidated_lock;
/**
* @userptr.invalidated: List of invalidated userptrs, not yet
* picked
* up for revalidation. Protected from access with the
* @invalidated_lock. Removing items from the list
* additionally requires @lock in write mode, and adding
* items to the list requires either the @svm.gpusvm.notifier_lock in
* write mode, OR @lock in write mode.
*/
struct list_head invalidated;
};
/** struct xe_userptr - User pointer */
struct xe_userptr {
/** @invalidate_link: Link for the vm::userptr.invalidated list */
struct list_head invalidate_link;
/** @userptr: link into VM repin list if userptr. */
struct list_head repin_link;
/**
* @pages: gpusvm pages for this user pointer.
*/
struct drm_gpusvm_pages pages;
/**
* @notifier: MMU notifier for user pointer (invalidation call back)
*/
struct mmu_interval_notifier notifier;
/**
* @finish: MMU notifier finish structure for two-pass invalidation.
* Embedded here to avoid allocation in the notifier callback.
* Protected by struct xe_vm::svm.gpusvm.notifier_lock in write mode
* alternatively by the same lock in read mode *and* the vm resv held.
*/
struct mmu_interval_notifier_finish finish;
/**
* @inval_batch: TLB invalidation batch for deferred completion.
* Stores an in-flight TLB invalidation submitted during a two-pass
* notifier so the wait can be deferred to a subsequent pass, allowing
* multiple GPUs to be signalled before any of them are waited on.
* Protected using the same locking as @finish.
*/
struct xe_tlb_inval_batch inval_batch;
/**
* @finish_inuse: Whether @finish is currently in use by an in-progress
* two-pass invalidation.
* Protected using the same locking as @finish.
*/
bool finish_inuse;
/**
* @tlb_inval_submitted: Whether a TLB invalidation has been submitted
* via @inval_batch and is pending completion. When set, the next pass
* must call xe_tlb_inval_batch_wait() before reusing @inval_batch.
* Protected using the same locking as @finish.
*/
bool tlb_inval_submitted;
/**
* @initial_bind: user pointer has been bound at least once.
* write: vm->svm.gpusvm.notifier_lock in read mode and vm->resv held.
* read: vm->svm.gpusvm.notifier_lock in write mode or vm->resv held.
*/
bool initial_bind;
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
u32 divisor;
#endif
};
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
void xe_userptr_remove(struct xe_userptr_vma *uvma);
int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start,
unsigned long range);
void xe_userptr_destroy(struct xe_userptr_vma *uvma);
int xe_vm_userptr_pin(struct xe_vm *vm);
int __xe_vm_userptr_needs_repin(struct xe_vm *vm);
int xe_vm_userptr_check_repin(struct xe_vm *vm);
int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma);
int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma);
#else
static inline void xe_userptr_remove(struct xe_userptr_vma *uvma) {}
static inline int xe_userptr_setup(struct xe_userptr_vma *uvma,
unsigned long start, unsigned long range)
{
return -ENODEV;
}
static inline void xe_userptr_destroy(struct xe_userptr_vma *uvma) {}
static inline int xe_vm_userptr_pin(struct xe_vm *vm) { return 0; }
static inline int __xe_vm_userptr_needs_repin(struct xe_vm *vm) { return 0; }
static inline int xe_vm_userptr_check_repin(struct xe_vm *vm) { return 0; }
static inline int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) { return -ENODEV; }
static inline int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma) { return -ENODEV; };
#endif
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma);
#else
static inline void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
{
}
#endif
#endif
|
