summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/core-api/dma-api.rst2
-rw-r--r--MAINTAINERS1
-rw-r--r--drivers/iommu/dma-iommu.c12
-rw-r--r--drivers/nvme/host/pci.c2
-rw-r--r--drivers/scsi/scsi_transport_sas.c12
-rw-r--r--include/linux/dma-map-ops.h4
-rw-r--r--include/linux/dma-mapping.h4
-rw-r--r--include/linux/gfp.h2
-rw-r--r--include/linux/iommu-dma.h2
-rw-r--r--kernel/dma/contiguous.c6
-rw-r--r--kernel/dma/direct.c8
-rw-r--r--kernel/dma/mapping.c10
-rw-r--r--kernel/dma/ops_helpers.c11
-rw-r--r--mm/page_alloc.c2
14 files changed, 43 insertions, 35 deletions
diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
index ba23a472f794..69d4ebe4f96d 100644
--- a/Documentation/core-api/dma-api.rst
+++ b/Documentation/core-api/dma-api.rst
@@ -154,7 +154,7 @@ others should not be larger than the returned value.
::
size_t
- dma_opt_mapping_size(struct device *dev);
+ dma_max_opt_mapping_size(struct device *dev);
Returns the maximum optimal size of a mapping for the device.
diff --git a/MAINTAINERS b/MAINTAINERS
index b5cf9c14f501..d392c95bcb60 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7730,6 +7730,7 @@ F: include/linux/dma-direct.h
F: include/linux/dma-map-ops.h
F: include/linux/dma-mapping.h
F: include/linux/swiotlb.h
+F: include/trace/events/dma.h
F: kernel/dma/
DMA MAPPING & SCATTERLIST API [RUST]
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 68ee57800fef..26b9c2d71cd3 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1628,10 +1628,12 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size,
void *cpu_addr;
page = dma_alloc_contiguous(dev, alloc_size, gfp);
- if (!page)
- page = alloc_pages_node(node, gfp, get_order(alloc_size));
- if (!page)
- return NULL;
+ if (!page) {
+ cpu_addr = alloc_pages_exact_nid(node, alloc_size, gfp);
+ if (!cpu_addr)
+ return NULL;
+ page = virt_to_page(cpu_addr);
+ }
if (!coherent || PageHighMem(page)) {
pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
@@ -1762,7 +1764,7 @@ unsigned long iommu_dma_get_merge_boundary(struct device *dev)
return (1UL << __ffs(domain->pgsize_bitmap)) - 1;
}
-size_t iommu_dma_opt_mapping_size(void)
+size_t iommu_dma_max_opt_mapping_size(void)
{
return iova_rcache_range();
}
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 5440cf18b55b..a953c0697f99 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -3747,7 +3747,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
*/
dev->ctrl.max_hw_sectors = min_t(u32,
NVME_MAX_BYTES >> SECTOR_SHIFT,
- dma_opt_mapping_size(&pdev->dev) >> 9);
+ dma_max_opt_mapping_size(&pdev->dev) >> SECTOR_SHIFT);
dev->ctrl.max_segments = NVME_MAX_SEGS;
dev->ctrl.max_integrity_segments = 1;
return dev;
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index d689b9ed08a6..aaa47552a215 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -223,8 +223,8 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
/*
* Set shost->opt_sectors from the DMA optimal mapping size, but only
- * when dma_opt_mapping_size() is strictly less than dma_max_mapping_size(),
- * indicating a genuine optimization hint from an IOMMU or DMA backend.
+ * when dma_max_opt_mapping_size() is strictly less than
+ * dma_max_mapping_size(), indicating a genuine optimization hint.
* When the two are equal (e.g. IOMMU disabled / passthrough), no real
* hint exists, so leave opt_sectors at 0 to avoid bogus optimal_io_size
* values that break filesystem geometry (e.g. mkfs.xfs stripe alignment).
@@ -232,16 +232,16 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
static void sas_dma_setup_opt_sectors(struct Scsi_Host *shost)
{
struct device *dma_dev = shost->dma_dev;
- size_t opt = dma_opt_mapping_size(dma_dev);
+ size_t max_opt = dma_max_opt_mapping_size(dma_dev);
size_t max = dma_max_mapping_size(dma_dev);
unsigned int opt_sectors;
- /* opt >= max means no real hint was provided by the DMA layer */
- if (opt >= max)
+ /* max_opt >= max means no real hint was provided by the DMA layer */
+ if (max_opt >= max)
return;
/* Clamp to max_sectors to avoid overflow in sector arithmetic */
- opt_sectors = min_t(unsigned int, opt >> SECTOR_SHIFT,
+ opt_sectors = min_t(unsigned int, max_opt >> SECTOR_SHIFT,
shost->max_sectors);
/* Guard against zero before rounddown_pow_of_two() */
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 8fae2b7deb20..afaf2c12189b 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -60,7 +60,7 @@ struct dma_map_ops {
int (*dma_supported)(struct device *dev, u64 mask);
u64 (*get_required_mask)(struct device *dev);
size_t (*max_mapping_size)(struct device *dev);
- size_t (*opt_mapping_size)(void);
+ size_t (*max_opt_mapping_size)(void);
unsigned long (*get_merge_boundary)(struct device *dev);
};
@@ -143,7 +143,7 @@ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size,
static inline void dma_free_contiguous(struct device *dev, struct page *page,
size_t size)
{
- __free_pages(page, get_order(size));
+ free_pages_exact(page_address(page), size);
}
#endif /* CONFIG_DMA_CMA*/
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index a3e880649fa4..efde0e5a58dc 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -207,7 +207,7 @@ int dma_set_coherent_mask(struct device *dev, u64 mask);
u64 dma_get_required_mask(struct device *dev);
bool dma_addressing_limited(struct device *dev);
size_t dma_max_mapping_size(struct device *dev);
-size_t dma_opt_mapping_size(struct device *dev);
+size_t dma_max_opt_mapping_size(struct device *dev);
unsigned long dma_get_merge_boundary(struct device *dev);
struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
enum dma_data_direction dir, gfp_t gfp, unsigned long attrs);
@@ -326,7 +326,7 @@ static inline size_t dma_max_mapping_size(struct device *dev)
{
return 0;
}
-static inline size_t dma_opt_mapping_size(struct device *dev)
+static inline size_t dma_max_opt_mapping_size(struct device *dev)
{
return 0;
}
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 872bc53f32ec..aa4601805f21 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -329,7 +329,7 @@ void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1);
void free_pages_exact(void *virt, size_t size);
-__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
+void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
#define alloc_pages_exact_nid(...) \
alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__))
diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h
index 060f6e23ab3c..fae5d50e4f27 100644
--- a/include/linux/iommu-dma.h
+++ b/include/linux/iommu-dma.h
@@ -39,7 +39,7 @@ int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs);
unsigned long iommu_dma_get_merge_boundary(struct device *dev);
-size_t iommu_dma_opt_mapping_size(void);
+size_t iommu_dma_max_opt_mapping_size(void);
size_t iommu_dma_max_mapping_size(struct device *dev);
void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t handle, unsigned long attrs);
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index 66093460584e..90937dc7c627 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -451,8 +451,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
*
* This function releases memory allocated by dma_alloc_contiguous(). As the
* cma_release returns false when provided pages do not belong to contiguous
- * area and true otherwise, this function then does a fallback __free_pages()
- * upon a false-return.
+ * area and true otherwise, this function then does a fallback
+ * free_pages_exact() upon a false-return.
*/
void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
{
@@ -476,7 +476,7 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
}
/* not in any cma, free from buddy */
- __free_pages(page, get_order(size));
+ free_pages_exact(page_address(page), size);
}
/*
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index da665ca22d5c..ea64453d7370 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -120,6 +120,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
int node = dev_to_node(dev);
struct page *page;
u64 phys_limit;
+ void *va;
WARN_ON_ONCE(!PAGE_ALIGNED(size));
@@ -133,9 +134,9 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
dma_free_contiguous(dev, page, size);
}
- while ((page = alloc_pages_node(node, gfp, get_order(size)))
- && !dma_coherent_ok(dev, page_to_phys(page), size)) {
- __free_pages(page, get_order(size));
+ while ((va = alloc_pages_exact_nid(node, size, gfp)) &&
+ !dma_coherent_ok(dev, virt_to_phys(va), size)) {
+ free_pages_exact(va, size);
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
phys_limit < DMA_BIT_MASK(64) &&
@@ -146,6 +147,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
else
return NULL;
}
+ page = va ? virt_to_page(va) : NULL;
return page;
}
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index bf2651a70b7c..82e701648d60 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -1012,19 +1012,19 @@ size_t dma_max_mapping_size(struct device *dev)
}
EXPORT_SYMBOL_GPL(dma_max_mapping_size);
-size_t dma_opt_mapping_size(struct device *dev)
+size_t dma_max_opt_mapping_size(struct device *dev)
{
const struct dma_map_ops *ops = get_dma_ops(dev);
size_t size = SIZE_MAX;
if (use_dma_iommu(dev))
- size = iommu_dma_opt_mapping_size();
- else if (ops && ops->opt_mapping_size)
- size = ops->opt_mapping_size();
+ size = iommu_dma_max_opt_mapping_size();
+ else if (ops && ops->max_opt_mapping_size)
+ size = ops->max_opt_mapping_size();
return min(dma_max_mapping_size(dev), size);
}
-EXPORT_SYMBOL_GPL(dma_opt_mapping_size);
+EXPORT_SYMBOL_GPL(dma_max_opt_mapping_size);
unsigned long dma_get_merge_boundary(struct device *dev)
{
diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c
index 6b5f9208d31c..5cdf3f60cf47 100644
--- a/kernel/dma/ops_helpers.c
+++ b/kernel/dma/ops_helpers.c
@@ -67,10 +67,13 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size,
phys_addr_t phys;
page = dma_alloc_contiguous(dev, size, gfp);
- if (!page)
- page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size));
- if (!page)
- return NULL;
+ if (!page) {
+ void *va = alloc_pages_exact_nid(dev_to_node(dev), size, gfp);
+
+ if (!va)
+ return NULL;
+ page = virt_to_page(va);
+ }
phys = page_to_phys(page);
if (use_dma_iommu(dev))
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 641384085e34..f13439d10b16 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5687,7 +5687,7 @@ EXPORT_SYMBOL(alloc_pages_exact_noprof);
*
* Return: pointer to the allocated area or %NULL in case of error.
*/
-void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask)
+void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask)
{
unsigned int order = get_order(size);
struct page *p;